diff --git a/lib/kokkos/Makefile.lammps b/lib/kokkos/Makefile.lammps new file mode 100755 index 0000000000..dd2af5ccfc --- /dev/null +++ b/lib/kokkos/Makefile.lammps @@ -0,0 +1,170 @@ +# This Makefile is intended to be include in an application Makefile. +# It will append the OBJ variable with objects which need to be build for Kokkos. +# It also will produce a KOKKOS_INC and a KOKKOS_LINK variable which must be +# appended to the compile and link flags of the application Makefile. +# Note that you cannot compile and link at the same time! +# If you want to include dependencies (i.e. trigger a rebuild of the application +# object files when Kokkos files change, you can include KOKKOS_HEADERS in your +# dependency list. +# The Makefile uses a number of variables which can be set on the commandline, or +# in the application Makefile prior to including this Makefile. These options set +# certain build options and are explained in the following. + +# Directory path to the Kokkos source directory (this could be the kokkos directory +# in the Trilinos git repository +KOKKOS_PATH ?= ../../lib/kokkos +# Directory paths to libraries potentially used by Kokkos (if the respective options +# are chosen) +CUDA_PATH ?= /usr/local/cuda +HWLOC_PATH ?= /usr/local/hwloc/default + +# Device options: enable Pthreads, OpenMP and/or CUDA device (if none is enabled +# the Serial device will be used) +PTHREADS ?= yes +OMP ?= yes +CUDA ?= no + +# Build for Debug mode: add debug flags and enable boundschecks within Kokkos +DEBUG ?= no + +# Code generation options: use AVX instruction set; build for Xeon Phi (MIC); use +# reduced precision math (sets compiler flags such --fast_math) +AVX ?= no +MIC ?= no +RED_PREC ?=no + +# Optional Libraries: use hwloc for thread affinity; use librt for timers +HWLOC ?= no +LIBRT ?= no + +# CUDA specific options: use UVM (requires CUDA 6+); use LDG loads instead of +# texture fetches; compile for relocatable device code (function pointers) +CUDA_UVM ?= no +CUDA_LDG ?= no +CUDA_RELOC ?= no + +# Settings for replacing generic linear algebra kernels of Kokkos with vendor +# libraries. +CUSPARSE ?= no +CUBLAS ?= no + +#Typically nothing should be changed after this point + +KOKKOS_INC = -I$(KOKKOS_PATH)/core/src -I$(KOKKOS_PATH)/containers/src -I$(KOKKOS_PATH)/algorithms/src -I$(KOKKOS_PATH)/linalg/src -I../ -DKOKKOS_DONT_INCLUDE_CORE_CONFIG_H + +KOKKOS_HEADERS = $(wildcard $(KOKKOS_PATH)/core/src/*.hpp) +KOKKOS_HEADERS += $(wildcard $(KOKKOS_PATH)/core/src/impl/*.hpp) +KOKKOS_HEADERS += $(wildcard $(KOKKOS_PATH)/containers/src/*.hpp) +KOKKOS_HEADERS += $(wildcard $(KOKKOS_PATH)/containers/src/impl/*.hpp) +KOKKOS_HEADERS += $(wildcard $(KOKKOS_PATH)/linalg/src/*.hpp) + +SRC_KOKKOS = $(wildcard $(KOKKOS_PATH)/core/src/impl/*.cpp) +SRC_KOKKOS += $(wildcard $(KOKKOS_PATH)/containers/src/impl/*.cpp) +KOKKOS_LIB = libkokkoscore.a + +ifeq ($(CUDA), yes) +KOKKOS_INC += -x cu -DKOKKOS_HAVE_CUDA +SRC_KOKKOS += $(wildcard $(KOKKOS_PATH)/core/src/Cuda/*.cpp) +SRC_KOKKOS += $(wildcard $(KOKKOS_PATH)/core/src/Cuda/*.cu) +KOKKOS_HEADERS += $(wildcard $(KOKKOS_PATH)/core/src/Cuda/*.hpp) +KOKKOS_LINK += -L$(CUDA_PATH)/lib64 -lcudart -lcuda +ifeq ($(CUDA_UVM), yes) +KOKKOS_INC += -DKOKKOS_USE_CUDA_UVM +endif +endif + +ifeq ($(CUSPARSE), yes) +KOKKOS_INC += -DKOKKOS_USE_CUSPARSE +KOKKOS_LIB += -lcusparse +endif + +ifeq ($(CUBLAS), yes) +KOKKOS_INC += -DKOKKOS_USE_CUBLAS +KOKKOS_LIB += -lcublas +endif + +ifeq ($(MIC), yes) +KOKKOS_INC += -mmic +KOKKOS_LINK += -mmic +AVX = no +endif + +ifeq ($(AVX), yes) +ifeq ($(CUDA), yes) +KOKKOS_INC += -Xcompiler -mavx +else +KOKKOS_INC += -mavx +endif +KOKKOS_LINK += -mavx +endif + +ifeq ($(PTHREADS),yes) +KOKKOS_INC += -DKOKKOS_HAVE_PTHREAD +KOKKOS_LIB += -lpthread +SRC_KOKKOS += $(wildcard $(KOKKOS_PATH)/core/src/Threads/*.cpp) +KOKKOS_HEADERS += $(wildcard $(KOKKOS_PATH)/core/src/Threads/*.hpp) +endif + +ifeq ($(OMP),yes) +KOKKOS_INC += -DKOKKOS_HAVE_OPENMP +SRC_KOKKOS += $(wildcard $(KOKKOS_PATH)/core/src/OpenMP/*.cpp) +KOKKOS_HEADERS += $(wildcard $(KOKKOS_PATH)/core/src/OpenMP/*.hpp) +ifeq ($(CUDA), yes) +KOKKOS_INC += -Xcompiler -fopenmp +else +KOKKOS_INC += -fopenmp +endif +KOKKOS_LINK += -fopenmp +endif + +ifeq ($(HWLOC),yes) +KOKKOS_INC += -DKOKKOS_HAVE_HWLOC -I$(HWLOC_PATH)/include +KOKKOS_LINK += -L$(HWLOC_PATH)/lib -lhwloc +endif + +ifeq ($(RED_PREC), yes) +KOKKOS_INC += --use_fast_math +endif + +ifeq ($(DEBUG), yes) +ifeq ($(CUDA), yes) +KOKKOS_INC += -G +endif +KOKKOS_INC += -g -DKOKKOS_EXPRESSION_CHECK -DENABLE_TRACEBACK +KOKKOS_LINK += -g -ldl +endif + +ifeq ($(LIBRT),yes) +KOKKOS_INC += -DKOKKOS_USE_LIBRT -DPREC_TIMER +KOKKOS_LIB += -lrt +endif + +ifeq ($(CUDA_LDG), yes) +KOKKOS_INC += -DKOKKOS_USE_LDG_INTRINSIC +endif + +ifeq ($(CUDA), yes) +ifeq ($(CUDA_RELOC), yes) +KOKKOS_INC += -DKOKKOS_CUDA_USE_RELOCATABLE_DEVICE_CODE --relocatable-device-code=true +KOKKOS_LINK += --relocatable-device-code=true +endif +endif + +# Must build with C++11 +KOKKOS_INC += --std=c++11 -DKOKKOS_HAVE_CXX11 + +OBJ_KOKKOS_TMP = $(SRC_KOKKOS:.cpp=.o) +OBJ_KOKKOS = $(OBJ_KOKKOS_TMP:.cu=.o) +OBJ_KOKKOS_LINK = $(notdir $(OBJ_KOKKOS)) + +override OBJ += kokkos_depend.o + +libkokkoscore.a: $(OBJ_KOKKOS) + ar cr libkokkoscore.a $(OBJ_KOKKOS_LINK) + +kokkos_depend.o: libkokkoscore.a + touch kokkos_depend.cpp + $(CC) $(CCFLAGS) $(SHFLAGS) $(EXTRA_INC) -c kokkos_depend.cpp + + +KOKKOS_LINK += -L./ $(KOKKOS_LIB) diff --git a/lib/kokkos/README b/lib/kokkos/README new file mode 100755 index 0000000000..59f5685bab --- /dev/null +++ b/lib/kokkos/README @@ -0,0 +1,44 @@ +Kokkos library + +Carter Edwards, Christian Trott, Daniel Sunderland +Sandia National Labs + +29 May 2014 +http://trilinos.sandia.gov/packages/kokkos/ + +------------------------- + +This directory has source files from the Kokkos library that LAMMPS +uses when building with its KOKKOS package. The package contains +versions of pair, fix, and atom styles written with Kokkos data +structures and calls to the Kokkos library that should run efficiently +on various kinds of accelerated nodes, including GPU and many-core +chips. + +Kokkos is a C++ library that provides two key abstractions for an +application like LAMMPS. First, it allows a single implementation of +an application kernel (e.g. a pair style) to run efficiently on +different kinds of hardware (GPU, Intel Phi, many-core chip). + +Second, it provides data abstractions to adjust (at compile time) the +memory layout of basic data structures like 2d and 3d arrays and allow +the transparent utilization of special hardware load and store units. +Such data structures are used in LAMMPS to store atom coordinates or +forces or neighbor lists. The layout is chosen to optimize +performance on different platforms. Again this operation is hidden +from the developer, and does not affect how the single implementation +of the kernel is coded. + +To build LAMMPS with Kokkos, you should not need to make any changes +to files in this directory. You can overrided defaults that are set +in Makefile.lammps when building LAMMPS, by defining variables as part +of the make command. Details of the build process with Kokkos are +explained in Section 2.3 of doc/Section_start.html. and in Section 5.9 +of doc/Section_accelerate.html. + +The one exception is that when using Kokkos with NVIDIA GPUs, the +CUDA_PATH setting in Makefile.lammps needs to point to the +installation of the Cuda software on your machine. The normal default +location is /usr/local/cuda. If this is not correct, you need to edit +Makefile.lammps. + diff --git a/lib/kokkos/TPL/KokkosTPL_dummy.cpp b/lib/kokkos/TPL/KokkosTPL_dummy.cpp new file mode 100755 index 0000000000..e69de29bb2 diff --git a/lib/kokkos/TPL/cmake/Dependencies.cmake b/lib/kokkos/TPL/cmake/Dependencies.cmake new file mode 100755 index 0000000000..7ea652bf32 --- /dev/null +++ b/lib/kokkos/TPL/cmake/Dependencies.cmake @@ -0,0 +1,9 @@ +SET(LIB_REQUIRED_DEP_PACKAGES) +SET(LIB_OPTIONAL_DEP_PACKAGES) +SET(TEST_REQUIRED_DEP_PACKAGES) +SET(TEST_OPTIONAL_DEP_PACKAGES) +SET(LIB_REQUIRED_DEP_TPLS) +# Only dependency: +SET(LIB_OPTIONAL_DEP_TPLS CUDA) +SET(TEST_REQUIRED_DEP_TPLS ) +SET(TEST_OPTIONAL_DEP_TPLS ) diff --git a/lib/kokkos/TPL/cub/block/block_discontinuity.cuh b/lib/kokkos/TPL/cub/block/block_discontinuity.cuh new file mode 100755 index 0000000000..76af003e58 --- /dev/null +++ b/lib/kokkos/TPL/cub/block/block_discontinuity.cuh @@ -0,0 +1,587 @@ +/****************************************************************************** + * Copyright (c) 2011, Duane Merrill. All rights reserved. + * Copyright (c) 2011-2013, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the NVIDIA CORPORATION nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + ******************************************************************************/ + +/** + * \file + * The cub::BlockDiscontinuity class provides [collective](index.html#sec0) methods for flagging discontinuities within an ordered set of items partitioned across a CUDA thread block. + */ + +#pragma once + +#include "../util_type.cuh" +#include "../util_namespace.cuh" + +/// Optional outer namespace(s) +CUB_NS_PREFIX + +/// CUB namespace +namespace cub { + +/** + * \brief The BlockDiscontinuity class provides [collective](index.html#sec0) methods for flagging discontinuities within an ordered set of items partitioned across a CUDA thread block. ![](discont_logo.png) + * \ingroup BlockModule + * + * \par Overview + * A set of "head flags" (or "tail flags") is often used to indicate corresponding items + * that differ from their predecessors (or successors). For example, head flags are convenient + * for demarcating disjoint data segments as part of a segmented scan or reduction. + * + * \tparam T The data type to be flagged. + * \tparam BLOCK_THREADS The thread block size in threads. + * + * \par A Simple Example + * \blockcollective{BlockDiscontinuity} + * \par + * The code snippet below illustrates the head flagging of 512 integer items that + * are partitioned in a [blocked arrangement](index.html#sec5sec4) across 128 threads + * where each thread owns 4 consecutive items. + * \par + * \code + * #include + * + * __global__ void ExampleKernel(...) + * { + * // Specialize BlockDiscontinuity for 128 threads on type int + * typedef cub::BlockDiscontinuity BlockDiscontinuity; + * + * // Allocate shared memory for BlockDiscontinuity + * __shared__ typename BlockDiscontinuity::TempStorage temp_storage; + * + * // Obtain a segment of consecutive items that are blocked across threads + * int thread_data[4]; + * ... + * + * // Collectively compute head flags for discontinuities in the segment + * int head_flags[4]; + * BlockDiscontinuity(temp_storage).FlagHeads(head_flags, thread_data, cub::Inequality()); + * + * \endcode + * \par + * Suppose the set of input \p thread_data across the block of threads is + * { [0,0,1,1], [1,1,1,1], [2,3,3,3], [3,4,4,4], ... }. + * The corresponding output \p head_flags in those threads will be + * { [1,0,1,0], [0,0,0,0], [1,1,0,0], [0,1,0,0], ... }. + * + * \par Performance Considerations + * - Zero bank conflicts for most types. + * + */ +template < + typename T, + int BLOCK_THREADS> +class BlockDiscontinuity +{ +private: + + /****************************************************************************** + * Type definitions + ******************************************************************************/ + + /// Shared memory storage layout type (last element from each thread's input) + typedef T _TempStorage[BLOCK_THREADS]; + + + /****************************************************************************** + * Utility methods + ******************************************************************************/ + + /// Internal storage allocator + __device__ __forceinline__ _TempStorage& PrivateStorage() + { + __shared__ _TempStorage private_storage; + return private_storage; + } + + + /// Specialization for when FlagOp has third index param + template ::HAS_PARAM> + struct ApplyOp + { + // Apply flag operator + static __device__ __forceinline__ bool Flag(FlagOp flag_op, const T &a, const T &b, int idx) + { + return flag_op(a, b, idx); + } + }; + + /// Specialization for when FlagOp does not have a third index param + template + struct ApplyOp + { + // Apply flag operator + static __device__ __forceinline__ bool Flag(FlagOp flag_op, const T &a, const T &b, int idx) + { + return flag_op(a, b); + } + }; + + + /****************************************************************************** + * Thread fields + ******************************************************************************/ + + /// Shared storage reference + _TempStorage &temp_storage; + + /// Linear thread-id + int linear_tid; + + +public: + + /// \smemstorage{BlockDiscontinuity} + struct TempStorage : Uninitialized<_TempStorage> {}; + + + /******************************************************************//** + * \name Collective constructors + *********************************************************************/ + //@{ + + /** + * \brief Collective constructor for 1D thread blocks using a private static allocation of shared memory as temporary storage. Threads are identified using threadIdx.x. + */ + __device__ __forceinline__ BlockDiscontinuity() + : + temp_storage(PrivateStorage()), + linear_tid(threadIdx.x) + {} + + + /** + * \brief Collective constructor for 1D thread blocks using the specified memory allocation as temporary storage. Threads are identified using threadIdx.x. + */ + __device__ __forceinline__ BlockDiscontinuity( + TempStorage &temp_storage) ///< [in] Reference to memory allocation having layout type TempStorage + : + temp_storage(temp_storage.Alias()), + linear_tid(threadIdx.x) + {} + + + /** + * \brief Collective constructor using a private static allocation of shared memory as temporary storage. Each thread is identified using the supplied linear thread identifier + */ + __device__ __forceinline__ BlockDiscontinuity( + int linear_tid) ///< [in] A suitable 1D thread-identifier for the calling thread (e.g., (threadIdx.y * blockDim.x) + linear_tid for 2D thread blocks) + : + temp_storage(PrivateStorage()), + linear_tid(linear_tid) + {} + + + /** + * \brief Collective constructor using the specified memory allocation as temporary storage. Each thread is identified using the supplied linear thread identifier. + */ + __device__ __forceinline__ BlockDiscontinuity( + TempStorage &temp_storage, ///< [in] Reference to memory allocation having layout type TempStorage + int linear_tid) ///< [in] [optional] A suitable 1D thread-identifier for the calling thread (e.g., (threadIdx.y * blockDim.x) + linear_tid for 2D thread blocks) + : + temp_storage(temp_storage.Alias()), + linear_tid(linear_tid) + {} + + + + //@} end member group + /******************************************************************//** + * \name Head flag operations + *********************************************************************/ + //@{ + + + /** + * \brief Sets head flags indicating discontinuities between items partitioned across the thread block, for which the first item has no reference and is always flagged. + * + * The flag head_flagsi is set for item + * inputi when + * flag_op(previous-item, inputi) + * returns \p true (where previous-item is either the preceding item + * in the same thread or the last item in the previous thread). + * Furthermore, head_flagsi is always set for + * input>0 in thread0. + * + * \blocked + * + * \smemreuse + * + * The code snippet below illustrates the head-flagging of 512 integer items that + * are partitioned in a [blocked arrangement](index.html#sec5sec4) across 128 threads + * where each thread owns 4 consecutive items. + * \par + * \code + * #include + * + * __global__ void ExampleKernel(...) + * { + * // Specialize BlockDiscontinuity for 128 threads on type int + * typedef cub::BlockDiscontinuity BlockDiscontinuity; + * + * // Allocate shared memory for BlockDiscontinuity + * __shared__ typename BlockDiscontinuity::TempStorage temp_storage; + * + * // Obtain a segment of consecutive items that are blocked across threads + * int thread_data[4]; + * ... + * + * // Collectively compute head flags for discontinuities in the segment + * int head_flags[4]; + * BlockDiscontinuity(temp_storage).FlagHeads(head_flags, thread_data, cub::Inequality()); + * + * \endcode + * \par + * Suppose the set of input \p thread_data across the block of threads is + * { [0,0,1,1], [1,1,1,1], [2,3,3,3], [3,4,4,4], ... }. + * The corresponding output \p head_flags in those threads will be + * { [1,0,1,0], [0,0,0,0], [1,1,0,0], [0,1,0,0], ... }. + * + * \tparam ITEMS_PER_THREAD [inferred] The number of consecutive items partitioned onto each thread. + * \tparam FlagT [inferred] The flag type (must be an integer type) + * \tparam FlagOp [inferred] Binary predicate functor type having member T operator()(const T &a, const T &b) or member T operator()(const T &a, const T &b, unsigned int b_index), and returning \p true if a discontinuity exists between \p a and \p b, otherwise \p false. \p b_index is the rank of b in the aggregate tile of data. + */ + template < + int ITEMS_PER_THREAD, + typename FlagT, + typename FlagOp> + __device__ __forceinline__ void FlagHeads( + FlagT (&head_flags)[ITEMS_PER_THREAD], ///< [out] Calling thread's discontinuity head_flags + T (&input)[ITEMS_PER_THREAD], ///< [in] Calling thread's input items + FlagOp flag_op) ///< [in] Binary boolean flag predicate + { + // Share last item + temp_storage[linear_tid] = input[ITEMS_PER_THREAD - 1]; + + __syncthreads(); + + // Set flag for first item + head_flags[0] = (linear_tid == 0) ? + 1 : // First thread + ApplyOp::Flag( + flag_op, + temp_storage[linear_tid - 1], + input[0], + linear_tid * ITEMS_PER_THREAD); + + // Set head_flags for remaining items + #pragma unroll + for (int ITEM = 1; ITEM < ITEMS_PER_THREAD; ITEM++) + { + head_flags[ITEM] = ApplyOp::Flag( + flag_op, + input[ITEM - 1], + input[ITEM], + (linear_tid * ITEMS_PER_THREAD) + ITEM); + } + } + + + /** + * \brief Sets head flags indicating discontinuities between items partitioned across the thread block. + * + * The flag head_flagsi is set for item + * inputi when + * flag_op(previous-item, inputi) + * returns \p true (where previous-item is either the preceding item + * in the same thread or the last item in the previous thread). + * For thread0, item input0 is compared + * against \p tile_predecessor_item. + * + * \blocked + * + * \smemreuse + * + * The code snippet below illustrates the head-flagging of 512 integer items that + * are partitioned in a [blocked arrangement](index.html#sec5sec4) across 128 threads + * where each thread owns 4 consecutive items. + * \par + * \code + * #include + * + * __global__ void ExampleKernel(...) + * { + * // Specialize BlockDiscontinuity for 128 threads on type int + * typedef cub::BlockDiscontinuity BlockDiscontinuity; + * + * // Allocate shared memory for BlockDiscontinuity + * __shared__ typename BlockDiscontinuity::TempStorage temp_storage; + * + * // Obtain a segment of consecutive items that are blocked across threads + * int thread_data[4]; + * ... + * + * // Have thread0 obtain the predecessor item for the entire tile + * int tile_predecessor_item; + * if (threadIdx.x == 0) tile_predecessor_item == ... + * + * // Collectively compute head flags for discontinuities in the segment + * int head_flags[4]; + * BlockDiscontinuity(temp_storage).FlagHeads( + * head_flags, thread_data, cub::Inequality(), tile_predecessor_item); + * + * \endcode + * \par + * Suppose the set of input \p thread_data across the block of threads is + * { [0,0,1,1], [1,1,1,1], [2,3,3,3], [3,4,4,4], ... }, + * and that \p tile_predecessor_item is \p 0. The corresponding output \p head_flags in those threads will be + * { [0,0,1,0], [0,0,0,0], [1,1,0,0], [0,1,0,0], ... }. + * + * \tparam ITEMS_PER_THREAD [inferred] The number of consecutive items partitioned onto each thread. + * \tparam FlagT [inferred] The flag type (must be an integer type) + * \tparam FlagOp [inferred] Binary predicate functor type having member T operator()(const T &a, const T &b) or member T operator()(const T &a, const T &b, unsigned int b_index), and returning \p true if a discontinuity exists between \p a and \p b, otherwise \p false. \p b_index is the rank of b in the aggregate tile of data. + */ + template < + int ITEMS_PER_THREAD, + typename FlagT, + typename FlagOp> + __device__ __forceinline__ void FlagHeads( + FlagT (&head_flags)[ITEMS_PER_THREAD], ///< [out] Calling thread's discontinuity head_flags + T (&input)[ITEMS_PER_THREAD], ///< [in] Calling thread's input items + FlagOp flag_op, ///< [in] Binary boolean flag predicate + T tile_predecessor_item) ///< [in] [thread0 only] Item with which to compare the first tile item (input0 from thread0). + { + // Share last item + temp_storage[linear_tid] = input[ITEMS_PER_THREAD - 1]; + + __syncthreads(); + + // Set flag for first item + int predecessor = (linear_tid == 0) ? + tile_predecessor_item : // First thread + temp_storage[linear_tid - 1]; + + head_flags[0] = ApplyOp::Flag( + flag_op, + predecessor, + input[0], + linear_tid * ITEMS_PER_THREAD); + + // Set flag for remaining items + #pragma unroll + for (int ITEM = 1; ITEM < ITEMS_PER_THREAD; ITEM++) + { + head_flags[ITEM] = ApplyOp::Flag( + flag_op, + input[ITEM - 1], + input[ITEM], + (linear_tid * ITEMS_PER_THREAD) + ITEM); + } + } + + + //@} end member group + /******************************************************************//** + * \name Tail flag operations + *********************************************************************/ + //@{ + + + /** + * \brief Sets tail flags indicating discontinuities between items partitioned across the thread block, for which the last item has no reference and is always flagged. + * + * The flag tail_flagsi is set for item + * inputi when + * flag_op(inputi, next-item) + * returns \p true (where next-item is either the next item + * in the same thread or the first item in the next thread). + * Furthermore, tail_flagsITEMS_PER_THREAD-1 is always + * set for threadBLOCK_THREADS-1. + * + * \blocked + * + * \smemreuse + * + * The code snippet below illustrates the tail-flagging of 512 integer items that + * are partitioned in a [blocked arrangement](index.html#sec5sec4) across 128 threads + * where each thread owns 4 consecutive items. + * \par + * \code + * #include + * + * __global__ void ExampleKernel(...) + * { + * // Specialize BlockDiscontinuity for 128 threads on type int + * typedef cub::BlockDiscontinuity BlockDiscontinuity; + * + * // Allocate shared memory for BlockDiscontinuity + * __shared__ typename BlockDiscontinuity::TempStorage temp_storage; + * + * // Obtain a segment of consecutive items that are blocked across threads + * int thread_data[4]; + * ... + * + * // Collectively compute tail flags for discontinuities in the segment + * int tail_flags[4]; + * BlockDiscontinuity(temp_storage).FlagTails(tail_flags, thread_data, cub::Inequality()); + * + * \endcode + * \par + * Suppose the set of input \p thread_data across the block of threads is + * { [0,0,1,1], [1,1,1,1], [2,3,3,3], ..., [124,125,125,125] }. + * The corresponding output \p tail_flags in those threads will be + * { [0,1,0,0], [0,0,0,1], [1,0,0,...], ..., [1,0,0,1] }. + * + * \tparam ITEMS_PER_THREAD [inferred] The number of consecutive items partitioned onto each thread. + * \tparam FlagT [inferred] The flag type (must be an integer type) + * \tparam FlagOp [inferred] Binary predicate functor type having member T operator()(const T &a, const T &b) or member T operator()(const T &a, const T &b, unsigned int b_index), and returning \p true if a discontinuity exists between \p a and \p b, otherwise \p false. \p b_index is the rank of b in the aggregate tile of data. + */ + template < + int ITEMS_PER_THREAD, + typename FlagT, + typename FlagOp> + __device__ __forceinline__ void FlagTails( + FlagT (&tail_flags)[ITEMS_PER_THREAD], ///< [out] Calling thread's discontinuity tail_flags + T (&input)[ITEMS_PER_THREAD], ///< [in] Calling thread's input items + FlagOp flag_op) ///< [in] Binary boolean flag predicate + { + // Share first item + temp_storage[linear_tid] = input[0]; + + __syncthreads(); + + // Set flag for last item + tail_flags[ITEMS_PER_THREAD - 1] = (linear_tid == BLOCK_THREADS - 1) ? + 1 : // Last thread + ApplyOp::Flag( + flag_op, + input[ITEMS_PER_THREAD - 1], + temp_storage[linear_tid + 1], + (linear_tid * ITEMS_PER_THREAD) + (ITEMS_PER_THREAD - 1)); + + // Set flags for remaining items + #pragma unroll + for (int ITEM = 0; ITEM < ITEMS_PER_THREAD - 1; ITEM++) + { + tail_flags[ITEM] = ApplyOp::Flag( + flag_op, + input[ITEM], + input[ITEM + 1], + (linear_tid * ITEMS_PER_THREAD) + ITEM); + } + } + + + /** + * \brief Sets tail flags indicating discontinuities between items partitioned across the thread block. + * + * The flag tail_flagsi is set for item + * inputi when + * flag_op(inputi, next-item) + * returns \p true (where next-item is either the next item + * in the same thread or the first item in the next thread). + * For threadBLOCK_THREADS-1, item + * inputITEMS_PER_THREAD-1 is compared + * against \p tile_predecessor_item. + * + * \blocked + * + * \smemreuse + * + * The code snippet below illustrates the tail-flagging of 512 integer items that + * are partitioned in a [blocked arrangement](index.html#sec5sec4) across 128 threads + * where each thread owns 4 consecutive items. + * \par + * \code + * #include + * + * __global__ void ExampleKernel(...) + * { + * // Specialize BlockDiscontinuity for 128 threads on type int + * typedef cub::BlockDiscontinuity BlockDiscontinuity; + * + * // Allocate shared memory for BlockDiscontinuity + * __shared__ typename BlockDiscontinuity::TempStorage temp_storage; + * + * // Obtain a segment of consecutive items that are blocked across threads + * int thread_data[4]; + * ... + * + * // Have thread127 obtain the successor item for the entire tile + * int tile_successor_item; + * if (threadIdx.x == 127) tile_successor_item == ... + * + * // Collectively compute tail flags for discontinuities in the segment + * int tail_flags[4]; + * BlockDiscontinuity(temp_storage).FlagTails( + * tail_flags, thread_data, cub::Inequality(), tile_successor_item); + * + * \endcode + * \par + * Suppose the set of input \p thread_data across the block of threads is + * { [0,0,1,1], [1,1,1,1], [2,3,3,3], ..., [124,125,125,125] } + * and that \p tile_successor_item is \p 125. The corresponding output \p tail_flags in those threads will be + * { [0,1,0,0], [0,0,0,1], [1,0,0,...], ..., [1,0,0,0] }. + * + * \tparam ITEMS_PER_THREAD [inferred] The number of consecutive items partitioned onto each thread. + * \tparam FlagT [inferred] The flag type (must be an integer type) + * \tparam FlagOp [inferred] Binary predicate functor type having member T operator()(const T &a, const T &b) or member T operator()(const T &a, const T &b, unsigned int b_index), and returning \p true if a discontinuity exists between \p a and \p b, otherwise \p false. \p b_index is the rank of b in the aggregate tile of data. + */ + template < + int ITEMS_PER_THREAD, + typename FlagT, + typename FlagOp> + __device__ __forceinline__ void FlagTails( + FlagT (&tail_flags)[ITEMS_PER_THREAD], ///< [out] Calling thread's discontinuity tail_flags + T (&input)[ITEMS_PER_THREAD], ///< [in] Calling thread's input items + FlagOp flag_op, ///< [in] Binary boolean flag predicate + T tile_successor_item) ///< [in] [threadBLOCK_THREADS-1 only] Item with which to compare the last tile item (inputITEMS_PER_THREAD-1 from threadBLOCK_THREADS-1). + { + // Share first item + temp_storage[linear_tid] = input[0]; + + __syncthreads(); + + // Set flag for last item + int successor_item = (linear_tid == BLOCK_THREADS - 1) ? + tile_successor_item : // Last thread + temp_storage[linear_tid + 1]; + + tail_flags[ITEMS_PER_THREAD - 1] = ApplyOp::Flag( + flag_op, + input[ITEMS_PER_THREAD - 1], + successor_item, + (linear_tid * ITEMS_PER_THREAD) + (ITEMS_PER_THREAD - 1)); + + // Set flags for remaining items + #pragma unroll + for (int ITEM = 0; ITEM < ITEMS_PER_THREAD - 1; ITEM++) + { + tail_flags[ITEM] = ApplyOp::Flag( + flag_op, + input[ITEM], + input[ITEM + 1], + (linear_tid * ITEMS_PER_THREAD) + ITEM); + } + } + + //@} end member group + +}; + + +} // CUB namespace +CUB_NS_POSTFIX // Optional outer namespace(s) diff --git a/lib/kokkos/TPL/cub/block/block_exchange.cuh b/lib/kokkos/TPL/cub/block/block_exchange.cuh new file mode 100755 index 0000000000..b7b95343b5 --- /dev/null +++ b/lib/kokkos/TPL/cub/block/block_exchange.cuh @@ -0,0 +1,918 @@ +/****************************************************************************** + * Copyright (c) 2011, Duane Merrill. All rights reserved. + * Copyright (c) 2011-2013, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the NVIDIA CORPORATION nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + ******************************************************************************/ + +/** + * \file + * The cub::BlockExchange class provides [collective](index.html#sec0) methods for rearranging data partitioned across a CUDA thread block. + */ + +#pragma once + +#include "../util_arch.cuh" +#include "../util_macro.cuh" +#include "../util_type.cuh" +#include "../util_namespace.cuh" + +/// Optional outer namespace(s) +CUB_NS_PREFIX + +/// CUB namespace +namespace cub { + +/** + * \brief The BlockExchange class provides [collective](index.html#sec0) methods for rearranging data partitioned across a CUDA thread block. ![](transpose_logo.png) + * \ingroup BlockModule + * + * \par Overview + * It is commonplace for blocks of threads to rearrange data items between + * threads. For example, the global memory subsystem prefers access patterns + * where data items are "striped" across threads (where consecutive threads access consecutive items), + * yet most block-wide operations prefer a "blocked" partitioning of items across threads + * (where consecutive items belong to a single thread). + * + * \par + * BlockExchange supports the following types of data exchanges: + * - Transposing between [blocked](index.html#sec5sec4) and [striped](index.html#sec5sec4) arrangements + * - Transposing between [blocked](index.html#sec5sec4) and [warp-striped](index.html#sec5sec4) arrangements + * - Scattering ranked items to a [blocked arrangement](index.html#sec5sec4) + * - Scattering ranked items to a [striped arrangement](index.html#sec5sec4) + * + * \tparam T The data type to be exchanged. + * \tparam BLOCK_THREADS The thread block size in threads. + * \tparam ITEMS_PER_THREAD The number of items partitioned onto each thread. + * \tparam WARP_TIME_SLICING [optional] When \p true, only use enough shared memory for a single warp's worth of tile data, time-slicing the block-wide exchange over multiple synchronized rounds. Yields a smaller memory footprint at the expense of decreased parallelism. (Default: false) + * + * \par A Simple Example + * \blockcollective{BlockExchange} + * \par + * The code snippet below illustrates the conversion from a "blocked" to a "striped" arrangement + * of 512 integer items partitioned across 128 threads where each thread owns 4 items. + * \par + * \code + * #include + * + * __global__ void ExampleKernel(int *d_data, ...) + * { + * // Specialize BlockExchange for 128 threads owning 4 integer items each + * typedef cub::BlockExchange BlockExchange; + * + * // Allocate shared memory for BlockExchange + * __shared__ typename BlockExchange::TempStorage temp_storage; + * + * // Load a tile of data striped across threads + * int thread_data[4]; + * cub::LoadStriped(threadIdx.x, d_data, thread_data); + * + * // Collectively exchange data into a blocked arrangement across threads + * BlockExchange(temp_storage).StripedToBlocked(thread_data); + * + * \endcode + * \par + * Suppose the set of striped input \p thread_data across the block of threads is + * { [0,128,256,384], [1,129,257,385], ..., [127,255,383,511] }. + * The corresponding output \p thread_data in those threads will be + * { [0,1,2,3], [4,5,6,7], [8,9,10,11], ..., [508,509,510,511] }. + * + * \par Performance Considerations + * - Proper device-specific padding ensures zero bank conflicts for most types. + * + */ +template < + typename T, + int BLOCK_THREADS, + int ITEMS_PER_THREAD, + bool WARP_TIME_SLICING = false> +class BlockExchange +{ +private: + + /****************************************************************************** + * Constants + ******************************************************************************/ + + enum + { + LOG_WARP_THREADS = PtxArchProps::LOG_WARP_THREADS, + WARP_THREADS = 1 << LOG_WARP_THREADS, + WARPS = (BLOCK_THREADS + PtxArchProps::WARP_THREADS - 1) / PtxArchProps::WARP_THREADS, + + LOG_SMEM_BANKS = PtxArchProps::LOG_SMEM_BANKS, + SMEM_BANKS = 1 << LOG_SMEM_BANKS, + + TILE_ITEMS = BLOCK_THREADS * ITEMS_PER_THREAD, + + TIME_SLICES = (WARP_TIME_SLICING) ? WARPS : 1, + + TIME_SLICED_THREADS = (WARP_TIME_SLICING) ? CUB_MIN(BLOCK_THREADS, WARP_THREADS) : BLOCK_THREADS, + TIME_SLICED_ITEMS = TIME_SLICED_THREADS * ITEMS_PER_THREAD, + + WARP_TIME_SLICED_THREADS = CUB_MIN(BLOCK_THREADS, WARP_THREADS), + WARP_TIME_SLICED_ITEMS = WARP_TIME_SLICED_THREADS * ITEMS_PER_THREAD, + + // Insert padding if the number of items per thread is a power of two + INSERT_PADDING = ((ITEMS_PER_THREAD & (ITEMS_PER_THREAD - 1)) == 0), + PADDING_ITEMS = (INSERT_PADDING) ? (TIME_SLICED_ITEMS >> LOG_SMEM_BANKS) : 0, + }; + + /****************************************************************************** + * Type definitions + ******************************************************************************/ + + /// Shared memory storage layout type + typedef T _TempStorage[TIME_SLICED_ITEMS + PADDING_ITEMS]; + +public: + + /// \smemstorage{BlockExchange} + struct TempStorage : Uninitialized<_TempStorage> {}; + +private: + + + /****************************************************************************** + * Thread fields + ******************************************************************************/ + + /// Shared storage reference + _TempStorage &temp_storage; + + /// Linear thread-id + int linear_tid; + int warp_lane; + int warp_id; + int warp_offset; + + + /****************************************************************************** + * Utility methods + ******************************************************************************/ + + /// Internal storage allocator + __device__ __forceinline__ _TempStorage& PrivateStorage() + { + __shared__ _TempStorage private_storage; + return private_storage; + } + + + /** + * Transposes data items from blocked arrangement to striped arrangement. Specialized for no timeslicing. + */ + __device__ __forceinline__ void BlockedToStriped( + T items[ITEMS_PER_THREAD], ///< [in-out] Items to exchange, converting between blocked and striped arrangements. + Int2Type time_slicing) + { + #pragma unroll + for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ITEM++) + { + int item_offset = (linear_tid * ITEMS_PER_THREAD) + ITEM; + if (INSERT_PADDING) item_offset += item_offset >> LOG_SMEM_BANKS; + temp_storage[item_offset] = items[ITEM]; + } + + __syncthreads(); + + #pragma unroll + for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ITEM++) + { + int item_offset = int(ITEM * BLOCK_THREADS) + linear_tid; + if (INSERT_PADDING) item_offset += item_offset >> LOG_SMEM_BANKS; + items[ITEM] = temp_storage[item_offset]; + } + } + + + /** + * Transposes data items from blocked arrangement to striped arrangement. Specialized for warp-timeslicing. + */ + __device__ __forceinline__ void BlockedToStriped( + T items[ITEMS_PER_THREAD], ///< [in-out] Items to exchange, converting between blocked and striped arrangements. + Int2Type time_slicing) + { + T temp_items[ITEMS_PER_THREAD]; + + #pragma unroll + for (int SLICE = 0; SLICE < TIME_SLICES; SLICE++) + { + const int SLICE_OFFSET = SLICE * TIME_SLICED_ITEMS; + const int SLICE_OOB = SLICE_OFFSET + TIME_SLICED_ITEMS; + + __syncthreads(); + + if (warp_id == SLICE) + { + #pragma unroll + for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ITEM++) + { + int item_offset = (warp_lane * ITEMS_PER_THREAD) + ITEM; + if (INSERT_PADDING) item_offset += item_offset >> LOG_SMEM_BANKS; + temp_storage[item_offset] = items[ITEM]; + } + } + + __syncthreads(); + + #pragma unroll + for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ITEM++) + { + // Read a strip of items + const int STRIP_OFFSET = ITEM * BLOCK_THREADS; + const int STRIP_OOB = STRIP_OFFSET + BLOCK_THREADS; + + if ((SLICE_OFFSET < STRIP_OOB) && (SLICE_OOB > STRIP_OFFSET)) + { + int item_offset = STRIP_OFFSET + linear_tid - SLICE_OFFSET; + if ((item_offset >= 0) && (item_offset < TIME_SLICED_ITEMS)) + { + if (INSERT_PADDING) item_offset += item_offset >> LOG_SMEM_BANKS; + temp_items[ITEM] = temp_storage[item_offset]; + } + } + } + } + + // Copy + #pragma unroll + for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ITEM++) + { + items[ITEM] = temp_items[ITEM]; + } + } + + + /** + * Transposes data items from blocked arrangement to warp-striped arrangement. Specialized for no timeslicing + */ + __device__ __forceinline__ void BlockedToWarpStriped( + T items[ITEMS_PER_THREAD], ///< [in-out] Items to exchange, converting between blocked and warp-striped arrangements. + Int2Type time_slicing) + { + #pragma unroll + for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ITEM++) + { + int item_offset = warp_offset + ITEM + (warp_lane * ITEMS_PER_THREAD); + if (INSERT_PADDING) item_offset += item_offset >> LOG_SMEM_BANKS; + temp_storage[item_offset] = items[ITEM]; + } + + #pragma unroll + for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ITEM++) + { + int item_offset = warp_offset + (ITEM * WARP_TIME_SLICED_THREADS) + warp_lane; + if (INSERT_PADDING) item_offset += item_offset >> LOG_SMEM_BANKS; + items[ITEM] = temp_storage[item_offset]; + } + } + + /** + * Transposes data items from blocked arrangement to warp-striped arrangement. Specialized for warp-timeslicing + */ + __device__ __forceinline__ void BlockedToWarpStriped( + T items[ITEMS_PER_THREAD], ///< [in-out] Items to exchange, converting between blocked and warp-striped arrangements. + Int2Type time_slicing) + { + #pragma unroll + for (int SLICE = 0; SLICE < TIME_SLICES; ++SLICE) + { + __syncthreads(); + + if (warp_id == SLICE) + { + #pragma unroll + for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ITEM++) + { + int item_offset = ITEM + (warp_lane * ITEMS_PER_THREAD); + if (INSERT_PADDING) item_offset += item_offset >> LOG_SMEM_BANKS; + temp_storage[item_offset] = items[ITEM]; + } + + #pragma unroll + for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ITEM++) + { + int item_offset = (ITEM * WARP_TIME_SLICED_THREADS) + warp_lane; + if (INSERT_PADDING) item_offset += item_offset >> LOG_SMEM_BANKS; + items[ITEM] = temp_storage[item_offset]; + } + } + } + } + + + /** + * Transposes data items from striped arrangement to blocked arrangement. Specialized for no timeslicing. + */ + __device__ __forceinline__ void StripedToBlocked( + T items[ITEMS_PER_THREAD], ///< [in-out] Items to exchange, converting between striped and blocked arrangements. + Int2Type time_slicing) + { + #pragma unroll + for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ITEM++) + { + int item_offset = int(ITEM * BLOCK_THREADS) + linear_tid; + if (INSERT_PADDING) item_offset += item_offset >> LOG_SMEM_BANKS; + temp_storage[item_offset] = items[ITEM]; + } + + __syncthreads(); + + // No timeslicing + #pragma unroll + for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ITEM++) + { + int item_offset = (linear_tid * ITEMS_PER_THREAD) + ITEM; + if (INSERT_PADDING) item_offset += item_offset >> LOG_SMEM_BANKS; + items[ITEM] = temp_storage[item_offset]; + } + } + + + /** + * Transposes data items from striped arrangement to blocked arrangement. Specialized for warp-timeslicing. + */ + __device__ __forceinline__ void StripedToBlocked( + T items[ITEMS_PER_THREAD], ///< [in-out] Items to exchange, converting between striped and blocked arrangements. + Int2Type time_slicing) + { + // Warp time-slicing + T temp_items[ITEMS_PER_THREAD]; + + #pragma unroll + for (int SLICE = 0; SLICE < TIME_SLICES; SLICE++) + { + const int SLICE_OFFSET = SLICE * TIME_SLICED_ITEMS; + const int SLICE_OOB = SLICE_OFFSET + TIME_SLICED_ITEMS; + + __syncthreads(); + + #pragma unroll + for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ITEM++) + { + // Write a strip of items + const int STRIP_OFFSET = ITEM * BLOCK_THREADS; + const int STRIP_OOB = STRIP_OFFSET + BLOCK_THREADS; + + if ((SLICE_OFFSET < STRIP_OOB) && (SLICE_OOB > STRIP_OFFSET)) + { + int item_offset = STRIP_OFFSET + linear_tid - SLICE_OFFSET; + if ((item_offset >= 0) && (item_offset < TIME_SLICED_ITEMS)) + { + if (INSERT_PADDING) item_offset += item_offset >> LOG_SMEM_BANKS; + temp_storage[item_offset] = items[ITEM]; + } + } + } + + __syncthreads(); + + if (warp_id == SLICE) + { + #pragma unroll + for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ITEM++) + { + int item_offset = (warp_lane * ITEMS_PER_THREAD) + ITEM; + if (INSERT_PADDING) item_offset += item_offset >> LOG_SMEM_BANKS; + temp_items[ITEM] = temp_storage[item_offset]; + } + } + } + + // Copy + #pragma unroll + for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ITEM++) + { + items[ITEM] = temp_items[ITEM]; + } + } + + + /** + * Transposes data items from warp-striped arrangement to blocked arrangement. Specialized for no timeslicing + */ + __device__ __forceinline__ void WarpStripedToBlocked( + T items[ITEMS_PER_THREAD], ///< [in-out] Items to exchange, converting between warp-striped and blocked arrangements. + Int2Type time_slicing) + { + #pragma unroll + for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ITEM++) + { + int item_offset = warp_offset + (ITEM * WARP_TIME_SLICED_THREADS) + warp_lane; + if (INSERT_PADDING) item_offset += item_offset >> LOG_SMEM_BANKS; + temp_storage[item_offset] = items[ITEM]; + } + + #pragma unroll + for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ITEM++) + { + int item_offset = warp_offset + ITEM + (warp_lane * ITEMS_PER_THREAD); + if (INSERT_PADDING) item_offset += item_offset >> LOG_SMEM_BANKS; + items[ITEM] = temp_storage[item_offset]; + } + } + + + /** + * Transposes data items from warp-striped arrangement to blocked arrangement. Specialized for warp-timeslicing + */ + __device__ __forceinline__ void WarpStripedToBlocked( + T items[ITEMS_PER_THREAD], ///< [in-out] Items to exchange, converting between warp-striped and blocked arrangements. + Int2Type time_slicing) + { + #pragma unroll + for (int SLICE = 0; SLICE < TIME_SLICES; ++SLICE) + { + __syncthreads(); + + if (warp_id == SLICE) + { + #pragma unroll + for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ITEM++) + { + int item_offset = (ITEM * WARP_TIME_SLICED_THREADS) + warp_lane; + if (INSERT_PADDING) item_offset += item_offset >> LOG_SMEM_BANKS; + temp_storage[item_offset] = items[ITEM]; + } + + #pragma unroll + for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ITEM++) + { + int item_offset = ITEM + (warp_lane * ITEMS_PER_THREAD); + if (INSERT_PADDING) item_offset += item_offset >> LOG_SMEM_BANKS; + items[ITEM] = temp_storage[item_offset]; + } + } + } + } + + + /** + * Exchanges data items annotated by rank into blocked arrangement. Specialized for no timeslicing. + */ + __device__ __forceinline__ void ScatterToBlocked( + T items[ITEMS_PER_THREAD], ///< [in-out] Items to exchange + int ranks[ITEMS_PER_THREAD], ///< [in] Corresponding scatter ranks + Int2Type time_slicing) + { + #pragma unroll + for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ITEM++) + { + int item_offset = ranks[ITEM]; + if (INSERT_PADDING) item_offset = SHR_ADD(item_offset, LOG_SMEM_BANKS, item_offset); + temp_storage[item_offset] = items[ITEM]; + } + + __syncthreads(); + + #pragma unroll + for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ITEM++) + { + int item_offset = (linear_tid * ITEMS_PER_THREAD) + ITEM; + if (INSERT_PADDING) item_offset = SHR_ADD(item_offset, LOG_SMEM_BANKS, item_offset); + items[ITEM] = temp_storage[item_offset]; + } + } + + /** + * Exchanges data items annotated by rank into blocked arrangement. Specialized for warp-timeslicing. + */ + __device__ __forceinline__ void ScatterToBlocked( + T items[ITEMS_PER_THREAD], ///< [in-out] Items to exchange + int ranks[ITEMS_PER_THREAD], ///< [in] Corresponding scatter ranks + Int2Type time_slicing) + { + T temp_items[ITEMS_PER_THREAD]; + + #pragma unroll + for (int SLICE = 0; SLICE < TIME_SLICES; SLICE++) + { + __syncthreads(); + + const int SLICE_OFFSET = TIME_SLICED_ITEMS * SLICE; + + #pragma unroll + for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ITEM++) + { + int item_offset = ranks[ITEM] - SLICE_OFFSET; + if ((item_offset >= 0) && (item_offset < WARP_TIME_SLICED_ITEMS)) + { + if (INSERT_PADDING) item_offset = SHR_ADD(item_offset, LOG_SMEM_BANKS, item_offset); + temp_storage[item_offset] = items[ITEM]; + } + } + + __syncthreads(); + + if (warp_id == SLICE) + { + #pragma unroll + for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ITEM++) + { + int item_offset = (warp_lane * ITEMS_PER_THREAD) + ITEM; + if (INSERT_PADDING) item_offset = SHR_ADD(item_offset, LOG_SMEM_BANKS, item_offset); + temp_items[ITEM] = temp_storage[item_offset]; + } + } + } + + // Copy + #pragma unroll + for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ITEM++) + { + items[ITEM] = temp_items[ITEM]; + } + } + + + /** + * Exchanges data items annotated by rank into striped arrangement. Specialized for no timeslicing. + */ + __device__ __forceinline__ void ScatterToStriped( + T items[ITEMS_PER_THREAD], ///< [in-out] Items to exchange + int ranks[ITEMS_PER_THREAD], ///< [in] Corresponding scatter ranks + Int2Type time_slicing) + { + #pragma unroll + for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ITEM++) + { + int item_offset = ranks[ITEM]; + if (INSERT_PADDING) item_offset = SHR_ADD(item_offset, LOG_SMEM_BANKS, item_offset); + temp_storage[item_offset] = items[ITEM]; + } + + __syncthreads(); + + #pragma unroll + for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ITEM++) + { + int item_offset = int(ITEM * BLOCK_THREADS) + linear_tid; + if (INSERT_PADDING) item_offset = SHR_ADD(item_offset, LOG_SMEM_BANKS, item_offset); + items[ITEM] = temp_storage[item_offset]; + } + } + + + /** + * Exchanges data items annotated by rank into striped arrangement. Specialized for warp-timeslicing. + */ + __device__ __forceinline__ void ScatterToStriped( + T items[ITEMS_PER_THREAD], ///< [in-out] Items to exchange + int ranks[ITEMS_PER_THREAD], ///< [in] Corresponding scatter ranks + Int2Type time_slicing) + { + T temp_items[ITEMS_PER_THREAD]; + + #pragma unroll + for (int SLICE = 0; SLICE < TIME_SLICES; SLICE++) + { + const int SLICE_OFFSET = SLICE * TIME_SLICED_ITEMS; + const int SLICE_OOB = SLICE_OFFSET + TIME_SLICED_ITEMS; + + __syncthreads(); + + #pragma unroll + for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ITEM++) + { + int item_offset = ranks[ITEM] - SLICE_OFFSET; + if ((item_offset >= 0) && (item_offset < WARP_TIME_SLICED_ITEMS)) + { + if (INSERT_PADDING) item_offset = SHR_ADD(item_offset, LOG_SMEM_BANKS, item_offset); + temp_storage[item_offset] = items[ITEM]; + } + } + + __syncthreads(); + + #pragma unroll + for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ITEM++) + { + // Read a strip of items + const int STRIP_OFFSET = ITEM * BLOCK_THREADS; + const int STRIP_OOB = STRIP_OFFSET + BLOCK_THREADS; + + if ((SLICE_OFFSET < STRIP_OOB) && (SLICE_OOB > STRIP_OFFSET)) + { + int item_offset = STRIP_OFFSET + linear_tid - SLICE_OFFSET; + if ((item_offset >= 0) && (item_offset < TIME_SLICED_ITEMS)) + { + if (INSERT_PADDING) item_offset += item_offset >> LOG_SMEM_BANKS; + temp_items[ITEM] = temp_storage[item_offset]; + } + } + } + } + + // Copy + #pragma unroll + for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ITEM++) + { + items[ITEM] = temp_items[ITEM]; + } + } + + +public: + + /******************************************************************//** + * \name Collective constructors + *********************************************************************/ + //@{ + + /** + * \brief Collective constructor for 1D thread blocks using a private static allocation of shared memory as temporary storage. Threads are identified using threadIdx.x. + */ + __device__ __forceinline__ BlockExchange() + : + temp_storage(PrivateStorage()), + linear_tid(threadIdx.x), + warp_lane(linear_tid & (WARP_THREADS - 1)), + warp_id(linear_tid >> LOG_WARP_THREADS), + warp_offset(warp_id * WARP_TIME_SLICED_ITEMS) + {} + + + /** + * \brief Collective constructor for 1D thread blocks using the specified memory allocation as temporary storage. Threads are identified using threadIdx.x. + */ + __device__ __forceinline__ BlockExchange( + TempStorage &temp_storage) ///< [in] Reference to memory allocation having layout type TempStorage + : + temp_storage(temp_storage.Alias()), + linear_tid(threadIdx.x), + warp_lane(linear_tid & (WARP_THREADS - 1)), + warp_id(linear_tid >> LOG_WARP_THREADS), + warp_offset(warp_id * WARP_TIME_SLICED_ITEMS) + {} + + + /** + * \brief Collective constructor using a private static allocation of shared memory as temporary storage. Each thread is identified using the supplied linear thread identifier + */ + __device__ __forceinline__ BlockExchange( + int linear_tid) ///< [in] A suitable 1D thread-identifier for the calling thread (e.g., (threadIdx.y * blockDim.x) + linear_tid for 2D thread blocks) + : + temp_storage(PrivateStorage()), + linear_tid(linear_tid), + warp_lane(linear_tid & (WARP_THREADS - 1)), + warp_id(linear_tid >> LOG_WARP_THREADS), + warp_offset(warp_id * WARP_TIME_SLICED_ITEMS) + {} + + + /** + * \brief Collective constructor using the specified memory allocation as temporary storage. Each thread is identified using the supplied linear thread identifier. + */ + __device__ __forceinline__ BlockExchange( + TempStorage &temp_storage, ///< [in] Reference to memory allocation having layout type TempStorage + int linear_tid) ///< [in] [optional] A suitable 1D thread-identifier for the calling thread (e.g., (threadIdx.y * blockDim.x) + linear_tid for 2D thread blocks) + : + temp_storage(temp_storage.Alias()), + linear_tid(linear_tid), + warp_lane(linear_tid & (WARP_THREADS - 1)), + warp_id(linear_tid >> LOG_WARP_THREADS), + warp_offset(warp_id * WARP_TIME_SLICED_ITEMS) + {} + + + //@} end member group + /******************************************************************//** + * \name Structured exchanges + *********************************************************************/ + //@{ + + /** + * \brief Transposes data items from striped arrangement to blocked arrangement. + * + * \smemreuse + * + * The code snippet below illustrates the conversion from a "striped" to a "blocked" arrangement + * of 512 integer items partitioned across 128 threads where each thread owns 4 items. + * \par + * \code + * #include + * + * __global__ void ExampleKernel(int *d_data, ...) + * { + * // Specialize BlockExchange for 128 threads owning 4 integer items each + * typedef cub::BlockExchange BlockExchange; + * + * // Allocate shared memory for BlockExchange + * __shared__ typename BlockExchange::TempStorage temp_storage; + * + * // Load a tile of ordered data into a striped arrangement across block threads + * int thread_data[4]; + * cub::LoadStriped(threadIdx.x, d_data, thread_data); + * + * // Collectively exchange data into a blocked arrangement across threads + * BlockExchange(temp_storage).StripedToBlocked(thread_data); + * + * \endcode + * \par + * Suppose the set of striped input \p thread_data across the block of threads is + * { [0,128,256,384], [1,129,257,385], ..., [127,255,383,511] } after loading from global memory. + * The corresponding output \p thread_data in those threads will be + * { [0,1,2,3], [4,5,6,7], [8,9,10,11], ..., [508,509,510,511] }. + * + */ + __device__ __forceinline__ void StripedToBlocked( + T items[ITEMS_PER_THREAD]) ///< [in-out] Items to exchange, converting between striped and blocked arrangements. + { + StripedToBlocked(items, Int2Type()); + } + + /** + * \brief Transposes data items from blocked arrangement to striped arrangement. + * + * \smemreuse + * + * The code snippet below illustrates the conversion from a "blocked" to a "striped" arrangement + * of 512 integer items partitioned across 128 threads where each thread owns 4 items. + * \par + * \code + * #include + * + * __global__ void ExampleKernel(int *d_data, ...) + * { + * // Specialize BlockExchange for 128 threads owning 4 integer items each + * typedef cub::BlockExchange BlockExchange; + * + * // Allocate shared memory for BlockExchange + * __shared__ typename BlockExchange::TempStorage temp_storage; + * + * // Obtain a segment of consecutive items that are blocked across threads + * int thread_data[4]; + * ... + * + * // Collectively exchange data into a striped arrangement across threads + * BlockExchange(temp_storage).BlockedToStriped(thread_data); + * + * // Store data striped across block threads into an ordered tile + * cub::StoreStriped(threadIdx.x, d_data, thread_data); + * + * \endcode + * \par + * Suppose the set of blocked input \p thread_data across the block of threads is + * { [0,1,2,3], [4,5,6,7], [8,9,10,11], ..., [508,509,510,511] }. + * The corresponding output \p thread_data in those threads will be + * { [0,128,256,384], [1,129,257,385], ..., [127,255,383,511] } in + * preparation for storing to global memory. + * + */ + __device__ __forceinline__ void BlockedToStriped( + T items[ITEMS_PER_THREAD]) ///< [in-out] Items to exchange, converting between blocked and striped arrangements. + { + BlockedToStriped(items, Int2Type()); + } + + + /** + * \brief Transposes data items from warp-striped arrangement to blocked arrangement. + * + * \smemreuse + * + * The code snippet below illustrates the conversion from a "warp-striped" to a "blocked" arrangement + * of 512 integer items partitioned across 128 threads where each thread owns 4 items. + * \par + * \code + * #include + * + * __global__ void ExampleKernel(int *d_data, ...) + * { + * // Specialize BlockExchange for 128 threads owning 4 integer items each + * typedef cub::BlockExchange BlockExchange; + * + * // Allocate shared memory for BlockExchange + * __shared__ typename BlockExchange::TempStorage temp_storage; + * + * // Load a tile of ordered data into a warp-striped arrangement across warp threads + * int thread_data[4]; + * cub::LoadSWarptriped(threadIdx.x, d_data, thread_data); + * + * // Collectively exchange data into a blocked arrangement across threads + * BlockExchange(temp_storage).WarpStripedToBlocked(thread_data); + * + * \endcode + * \par + * Suppose the set of warp-striped input \p thread_data across the block of threads is + * { [0,32,64,96], [1,33,65,97], [2,34,66,98], ..., [415,447,479,511] } + * after loading from global memory. (The first 128 items are striped across + * the first warp of 32 threads, the second 128 items are striped across the second warp, etc.) + * The corresponding output \p thread_data in those threads will be + * { [0,1,2,3], [4,5,6,7], [8,9,10,11], ..., [508,509,510,511] }. + * + */ + __device__ __forceinline__ void WarpStripedToBlocked( + T items[ITEMS_PER_THREAD]) ///< [in-out] Items to exchange, converting between warp-striped and blocked arrangements. + { + WarpStripedToBlocked(items, Int2Type()); + } + + /** + * \brief Transposes data items from blocked arrangement to warp-striped arrangement. + * + * \smemreuse + * + * The code snippet below illustrates the conversion from a "blocked" to a "warp-striped" arrangement + * of 512 integer items partitioned across 128 threads where each thread owns 4 items. + * \par + * \code + * #include + * + * __global__ void ExampleKernel(int *d_data, ...) + * { + * // Specialize BlockExchange for 128 threads owning 4 integer items each + * typedef cub::BlockExchange BlockExchange; + * + * // Allocate shared memory for BlockExchange + * __shared__ typename BlockExchange::TempStorage temp_storage; + * + * // Obtain a segment of consecutive items that are blocked across threads + * int thread_data[4]; + * ... + * + * // Collectively exchange data into a warp-striped arrangement across threads + * BlockExchange(temp_storage).BlockedToWarpStriped(thread_data); + * + * // Store data striped across warp threads into an ordered tile + * cub::StoreStriped(threadIdx.x, d_data, thread_data); + * + * \endcode + * \par + * Suppose the set of blocked input \p thread_data across the block of threads is + * { [0,1,2,3], [4,5,6,7], [8,9,10,11], ..., [508,509,510,511] }. + * The corresponding output \p thread_data in those threads will be + * { [0,32,64,96], [1,33,65,97], [2,34,66,98], ..., [415,447,479,511] } + * in preparation for storing to global memory. (The first 128 items are striped across + * the first warp of 32 threads, the second 128 items are striped across the second warp, etc.) + * + */ + __device__ __forceinline__ void BlockedToWarpStriped( + T items[ITEMS_PER_THREAD]) ///< [in-out] Items to exchange, converting between blocked and warp-striped arrangements. + { + BlockedToWarpStriped(items, Int2Type()); + } + + + //@} end member group + /******************************************************************//** + * \name Scatter exchanges + *********************************************************************/ + //@{ + + + /** + * \brief Exchanges data items annotated by rank into blocked arrangement. + * + * \smemreuse + */ + __device__ __forceinline__ void ScatterToBlocked( + T items[ITEMS_PER_THREAD], ///< [in-out] Items to exchange + int ranks[ITEMS_PER_THREAD]) ///< [in] Corresponding scatter ranks + { + ScatterToBlocked(items, ranks, Int2Type()); + } + + + /** + * \brief Exchanges data items annotated by rank into striped arrangement. + * + * \smemreuse + */ + __device__ __forceinline__ void ScatterToStriped( + T items[ITEMS_PER_THREAD], ///< [in-out] Items to exchange + int ranks[ITEMS_PER_THREAD]) ///< [in] Corresponding scatter ranks + { + ScatterToStriped(items, ranks, Int2Type()); + } + + //@} end member group + + +}; + +} // CUB namespace +CUB_NS_POSTFIX // Optional outer namespace(s) + diff --git a/lib/kokkos/TPL/cub/block/block_histogram.cuh b/lib/kokkos/TPL/cub/block/block_histogram.cuh new file mode 100755 index 0000000000..dd346e3954 --- /dev/null +++ b/lib/kokkos/TPL/cub/block/block_histogram.cuh @@ -0,0 +1,414 @@ +/****************************************************************************** + * Copyright (c) 2011, Duane Merrill. All rights reserved. + * Copyright (c) 2011-2013, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the NVIDIA CORPORATION nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + ******************************************************************************/ + +/** + * \file + * The cub::BlockHistogram class provides [collective](index.html#sec0) methods for constructing block-wide histograms from data samples partitioned across a CUDA thread block. + */ + +#pragma once + +#include "specializations/block_histogram_sort.cuh" +#include "specializations/block_histogram_atomic.cuh" +#include "../util_arch.cuh" +#include "../util_namespace.cuh" + +/// Optional outer namespace(s) +CUB_NS_PREFIX + +/// CUB namespace +namespace cub { + + +/****************************************************************************** + * Algorithmic variants + ******************************************************************************/ + +/** + * \brief BlockHistogramAlgorithm enumerates alternative algorithms for the parallel construction of block-wide histograms. + */ +enum BlockHistogramAlgorithm +{ + + /** + * \par Overview + * Sorting followed by differentiation. Execution is comprised of two phases: + * -# Sort the data using efficient radix sort + * -# Look for "runs" of same-valued keys by detecting discontinuities; the run-lengths are histogram bin counts. + * + * \par Performance Considerations + * Delivers consistent throughput regardless of sample bin distribution. + */ + BLOCK_HISTO_SORT, + + + /** + * \par Overview + * Use atomic addition to update byte counts directly + * + * \par Performance Considerations + * Performance is strongly tied to the hardware implementation of atomic + * addition, and may be significantly degraded for non uniformly-random + * input distributions where many concurrent updates are likely to be + * made to the same bin counter. + */ + BLOCK_HISTO_ATOMIC, +}; + + + +/****************************************************************************** + * Block histogram + ******************************************************************************/ + + +/** + * \brief The BlockHistogram class provides [collective](index.html#sec0) methods for constructing block-wide histograms from data samples partitioned across a CUDA thread block. ![](histogram_logo.png) + * \ingroup BlockModule + * + * \par Overview + * A histogram + * counts the number of observations that fall into each of the disjoint categories (known as bins). + * + * \par + * Optionally, BlockHistogram can be specialized to use different algorithms: + * -# cub::BLOCK_HISTO_SORT. Sorting followed by differentiation. [More...](\ref cub::BlockHistogramAlgorithm) + * -# cub::BLOCK_HISTO_ATOMIC. Use atomic addition to update byte counts directly. [More...](\ref cub::BlockHistogramAlgorithm) + * + * \tparam T The sample type being histogrammed (must be castable to an integer bin identifier) + * \tparam BLOCK_THREADS The thread block size in threads + * \tparam ITEMS_PER_THREAD The number of items per thread + * \tparam BINS The number bins within the histogram + * \tparam ALGORITHM [optional] cub::BlockHistogramAlgorithm enumerator specifying the underlying algorithm to use (default: cub::BLOCK_HISTO_SORT) + * + * \par A Simple Example + * \blockcollective{BlockHistogram} + * \par + * The code snippet below illustrates a 256-bin histogram of 512 integer samples that + * are partitioned across 128 threads where each thread owns 4 samples. + * \par + * \code + * #include + * + * __global__ void ExampleKernel(...) + * { + * // Specialize a 256-bin BlockHistogram type for 128 threads having 4 character samples each + * typedef cub::BlockHistogram BlockHistogram; + * + * // Allocate shared memory for BlockHistogram + * __shared__ typename BlockHistogram::TempStorage temp_storage; + * + * // Allocate shared memory for block-wide histogram bin counts + * __shared__ unsigned int smem_histogram[256]; + * + * // Obtain input samples per thread + * unsigned char data[4]; + * ... + * + * // Compute the block-wide histogram + * BlockHistogram(temp_storage).Histogram(data, smem_histogram); + * + * \endcode + * + * \par Performance and Usage Considerations + * - The histogram output can be constructed in shared or global memory + * - See cub::BlockHistogramAlgorithm for performance details regarding algorithmic alternatives + * + */ +template < + typename T, + int BLOCK_THREADS, + int ITEMS_PER_THREAD, + int BINS, + BlockHistogramAlgorithm ALGORITHM = BLOCK_HISTO_SORT> +class BlockHistogram +{ +private: + + /****************************************************************************** + * Constants and type definitions + ******************************************************************************/ + + /** + * Ensure the template parameterization meets the requirements of the + * targeted device architecture. BLOCK_HISTO_ATOMIC can only be used + * on version SM120 or later. Otherwise BLOCK_HISTO_SORT is used + * regardless. + */ + static const BlockHistogramAlgorithm SAFE_ALGORITHM = + ((ALGORITHM == BLOCK_HISTO_ATOMIC) && (CUB_PTX_ARCH < 120)) ? + BLOCK_HISTO_SORT : + ALGORITHM; + + /// Internal specialization. + typedef typename If<(SAFE_ALGORITHM == BLOCK_HISTO_SORT), + BlockHistogramSort, + BlockHistogramAtomic >::Type InternalBlockHistogram; + + /// Shared memory storage layout type for BlockHistogram + typedef typename InternalBlockHistogram::TempStorage _TempStorage; + + + /****************************************************************************** + * Thread fields + ******************************************************************************/ + + /// Shared storage reference + _TempStorage &temp_storage; + + /// Linear thread-id + int linear_tid; + + + /****************************************************************************** + * Utility methods + ******************************************************************************/ + + /// Internal storage allocator + __device__ __forceinline__ _TempStorage& PrivateStorage() + { + __shared__ _TempStorage private_storage; + return private_storage; + } + + +public: + + /// \smemstorage{BlockHistogram} + struct TempStorage : Uninitialized<_TempStorage> {}; + + + /******************************************************************//** + * \name Collective constructors + *********************************************************************/ + //@{ + + /** + * \brief Collective constructor for 1D thread blocks using a private static allocation of shared memory as temporary storage. Threads are identified using threadIdx.x. + */ + __device__ __forceinline__ BlockHistogram() + : + temp_storage(PrivateStorage()), + linear_tid(threadIdx.x) + {} + + + /** + * \brief Collective constructor for 1D thread blocks using the specified memory allocation as temporary storage. Threads are identified using threadIdx.x. + */ + __device__ __forceinline__ BlockHistogram( + TempStorage &temp_storage) ///< [in] Reference to memory allocation having layout type TempStorage + : + temp_storage(temp_storage.Alias()), + linear_tid(threadIdx.x) + {} + + + /** + * \brief Collective constructor using a private static allocation of shared memory as temporary storage. Each thread is identified using the supplied linear thread identifier + */ + __device__ __forceinline__ BlockHistogram( + int linear_tid) ///< [in] A suitable 1D thread-identifier for the calling thread (e.g., (threadIdx.y * blockDim.x) + linear_tid for 2D thread blocks) + : + temp_storage(PrivateStorage()), + linear_tid(linear_tid) + {} + + + /** + * \brief Collective constructor using the specified memory allocation as temporary storage. Each thread is identified using the supplied linear thread identifier. + */ + __device__ __forceinline__ BlockHistogram( + TempStorage &temp_storage, ///< [in] Reference to memory allocation having layout type TempStorage + int linear_tid) ///< [in] [optional] A suitable 1D thread-identifier for the calling thread (e.g., (threadIdx.y * blockDim.x) + linear_tid for 2D thread blocks) + : + temp_storage(temp_storage.Alias()), + linear_tid(linear_tid) + {} + + + //@} end member group + /******************************************************************//** + * \name Histogram operations + *********************************************************************/ + //@{ + + + /** + * \brief Initialize the shared histogram counters to zero. + * + * The code snippet below illustrates a the initialization and update of a + * histogram of 512 integer samples that are partitioned across 128 threads + * where each thread owns 4 samples. + * \par + * \code + * #include + * + * __global__ void ExampleKernel(...) + * { + * // Specialize a 256-bin BlockHistogram type for 128 threads having 4 character samples each + * typedef cub::BlockHistogram BlockHistogram; + * + * // Allocate shared memory for BlockHistogram + * __shared__ typename BlockHistogram::TempStorage temp_storage; + * + * // Allocate shared memory for block-wide histogram bin counts + * __shared__ unsigned int smem_histogram[256]; + * + * // Obtain input samples per thread + * unsigned char thread_samples[4]; + * ... + * + * // Initialize the block-wide histogram + * BlockHistogram(temp_storage).InitHistogram(smem_histogram); + * + * // Update the block-wide histogram + * BlockHistogram(temp_storage).Composite(thread_samples, smem_histogram); + * + * \endcode + * + * \tparam HistoCounter [inferred] Histogram counter type + */ + template + __device__ __forceinline__ void InitHistogram(HistoCounter histogram[BINS]) + { + // Initialize histogram bin counts to zeros + int histo_offset = 0; + + #pragma unroll + for(; histo_offset + BLOCK_THREADS <= BINS; histo_offset += BLOCK_THREADS) + { + histogram[histo_offset + linear_tid] = 0; + } + // Finish up with guarded initialization if necessary + if ((BINS % BLOCK_THREADS != 0) && (histo_offset + linear_tid < BINS)) + { + histogram[histo_offset + linear_tid] = 0; + } + } + + + /** + * \brief Constructs a block-wide histogram in shared/global memory. Each thread contributes an array of input elements. + * + * \smemreuse + * + * The code snippet below illustrates a 256-bin histogram of 512 integer samples that + * are partitioned across 128 threads where each thread owns 4 samples. + * \par + * \code + * #include + * + * __global__ void ExampleKernel(...) + * { + * // Specialize a 256-bin BlockHistogram type for 128 threads having 4 character samples each + * typedef cub::BlockHistogram BlockHistogram; + * + * // Allocate shared memory for BlockHistogram + * __shared__ typename BlockHistogram::TempStorage temp_storage; + * + * // Allocate shared memory for block-wide histogram bin counts + * __shared__ unsigned int smem_histogram[256]; + * + * // Obtain input samples per thread + * unsigned char thread_samples[4]; + * ... + * + * // Compute the block-wide histogram + * BlockHistogram(temp_storage).Histogram(thread_samples, smem_histogram); + * + * \endcode + * + * \tparam HistoCounter [inferred] Histogram counter type + */ + template < + typename HistoCounter> + __device__ __forceinline__ void Histogram( + T (&items)[ITEMS_PER_THREAD], ///< [in] Calling thread's input values to histogram + HistoCounter histogram[BINS]) ///< [out] Reference to shared/global memory histogram + { + // Initialize histogram bin counts to zeros + InitHistogram(histogram); + + // Composite the histogram + InternalBlockHistogram(temp_storage, linear_tid).Composite(items, histogram); + } + + + + /** + * \brief Updates an existing block-wide histogram in shared/global memory. Each thread composites an array of input elements. + * + * \smemreuse + * + * The code snippet below illustrates a the initialization and update of a + * histogram of 512 integer samples that are partitioned across 128 threads + * where each thread owns 4 samples. + * \par + * \code + * #include + * + * __global__ void ExampleKernel(...) + * { + * // Specialize a 256-bin BlockHistogram type for 128 threads having 4 character samples each + * typedef cub::BlockHistogram BlockHistogram; + * + * // Allocate shared memory for BlockHistogram + * __shared__ typename BlockHistogram::TempStorage temp_storage; + * + * // Allocate shared memory for block-wide histogram bin counts + * __shared__ unsigned int smem_histogram[256]; + * + * // Obtain input samples per thread + * unsigned char thread_samples[4]; + * ... + * + * // Initialize the block-wide histogram + * BlockHistogram(temp_storage).InitHistogram(smem_histogram); + * + * // Update the block-wide histogram + * BlockHistogram(temp_storage).Composite(thread_samples, smem_histogram); + * + * \endcode + * + * \tparam HistoCounter [inferred] Histogram counter type + */ + template < + typename HistoCounter> + __device__ __forceinline__ void Composite( + T (&items)[ITEMS_PER_THREAD], ///< [in] Calling thread's input values to histogram + HistoCounter histogram[BINS]) ///< [out] Reference to shared/global memory histogram + { + InternalBlockHistogram(temp_storage, linear_tid).Composite(items, histogram); + } + +}; + +} // CUB namespace +CUB_NS_POSTFIX // Optional outer namespace(s) + diff --git a/lib/kokkos/TPL/cub/block/block_load.cuh b/lib/kokkos/TPL/cub/block/block_load.cuh new file mode 100755 index 0000000000..e645bcdce9 --- /dev/null +++ b/lib/kokkos/TPL/cub/block/block_load.cuh @@ -0,0 +1,1122 @@ +/****************************************************************************** + * Copyright (c) 2011, Duane Merrill. All rights reserved. + * Copyright (c) 2011-2013, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the NVIDIA CORPORATION nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + ******************************************************************************/ + +/** + * \file + * Operations for reading linear tiles of data into the CUDA thread block. + */ + +#pragma once + +#include + +#include "../util_namespace.cuh" +#include "../util_macro.cuh" +#include "../util_type.cuh" +#include "../util_vector.cuh" +#include "../thread/thread_load.cuh" +#include "block_exchange.cuh" + +/// Optional outer namespace(s) +CUB_NS_PREFIX + +/// CUB namespace +namespace cub { + +/** + * \addtogroup IoModule + * @{ + */ + + +/******************************************************************//** + * \name Blocked I/O + *********************************************************************/ +//@{ + + +/** + * \brief Load a linear segment of items into a blocked arrangement across the thread block using the specified cache modifier. + * + * \blocked + * + * \tparam MODIFIER cub::PtxLoadModifier cache modifier. + * \tparam T [inferred] The data type to load. + * \tparam ITEMS_PER_THREAD [inferred] The number of consecutive items partitioned onto each thread. + * \tparam InputIteratorRA [inferred] The random-access iterator type for input (may be a simple pointer type). + */ +template < + PtxLoadModifier MODIFIER, + typename T, + int ITEMS_PER_THREAD, + typename InputIteratorRA> +__device__ __forceinline__ void LoadBlocked( + int linear_tid, ///< [in] A suitable 1D thread-identifier for the calling thread (e.g., (threadIdx.y * blockDim.x) + linear_tid for 2D thread blocks) + InputIteratorRA block_itr, ///< [in] The thread block's base input iterator for loading from + T (&items)[ITEMS_PER_THREAD]) ///< [out] Data to load +{ + // Load directly in thread-blocked order + #pragma unroll + for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ITEM++) + { + items[ITEM] = ThreadLoad(block_itr + (linear_tid * ITEMS_PER_THREAD) + ITEM); + } +} + + +/** + * \brief Load a linear segment of items into a blocked arrangement across the thread block using the specified cache modifier, guarded by range. + * + * \blocked + * + * \tparam MODIFIER cub::PtxLoadModifier cache modifier. + * \tparam T [inferred] The data type to load. + * \tparam ITEMS_PER_THREAD [inferred] The number of consecutive items partitioned onto each thread. + * \tparam InputIteratorRA [inferred] The random-access iterator type for input (may be a simple pointer type). + */ +template < + PtxLoadModifier MODIFIER, + typename T, + int ITEMS_PER_THREAD, + typename InputIteratorRA> +__device__ __forceinline__ void LoadBlocked( + int linear_tid, ///< [in] A suitable 1D thread-identifier for the calling thread (e.g., (threadIdx.y * blockDim.x) + linear_tid for 2D thread blocks) + InputIteratorRA block_itr, ///< [in] The thread block's base input iterator for loading from + T (&items)[ITEMS_PER_THREAD], ///< [out] Data to load + int valid_items) ///< [in] Number of valid items to load +{ + int bounds = valid_items - (linear_tid * ITEMS_PER_THREAD); + + #pragma unroll + for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ITEM++) + { + if (ITEM < bounds) + { + items[ITEM] = ThreadLoad(block_itr + (linear_tid * ITEMS_PER_THREAD) + ITEM); + } + } +} + + +/** + * \brief Load a linear segment of items into a blocked arrangement across the thread block using the specified cache modifier, guarded by range, with a fall-back assignment of out-of-bound elements.. + * + * \blocked + * + * \tparam MODIFIER cub::PtxLoadModifier cache modifier. + * \tparam T [inferred] The data type to load. + * \tparam ITEMS_PER_THREAD [inferred] The number of consecutive items partitioned onto each thread. + * \tparam InputIteratorRA [inferred] The random-access iterator type for input (may be a simple pointer type). + */ +template < + PtxLoadModifier MODIFIER, + typename T, + int ITEMS_PER_THREAD, + typename InputIteratorRA> +__device__ __forceinline__ void LoadBlocked( + int linear_tid, ///< [in] A suitable 1D thread-identifier for the calling thread (e.g., (threadIdx.y * blockDim.x) + linear_tid for 2D thread blocks) + InputIteratorRA block_itr, ///< [in] The thread block's base input iterator for loading from + T (&items)[ITEMS_PER_THREAD], ///< [out] Data to load + int valid_items, ///< [in] Number of valid items to load + T oob_default) ///< [in] Default value to assign out-of-bound items +{ + int bounds = valid_items - (linear_tid * ITEMS_PER_THREAD); + + #pragma unroll + for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ITEM++) + { + items[ITEM] = (ITEM < bounds) ? + ThreadLoad(block_itr + (linear_tid * ITEMS_PER_THREAD) + ITEM) : + oob_default; + } +} + + + +//@} end member group +/******************************************************************//** + * \name Striped I/O + *********************************************************************/ +//@{ + + +/** + * \brief Load a linear segment of items into a striped arrangement across the thread block using the specified cache modifier. + * + * \striped + * + * \tparam MODIFIER cub::PtxLoadModifier cache modifier. + * \tparam BLOCK_THREADS The thread block size in threads + * \tparam T [inferred] The data type to load. + * \tparam ITEMS_PER_THREAD [inferred] The number of consecutive items partitioned onto each thread. + * \tparam InputIteratorRA [inferred] The random-access iterator type for input (may be a simple pointer type). + */ +template < + PtxLoadModifier MODIFIER, + int BLOCK_THREADS, + typename T, + int ITEMS_PER_THREAD, + typename InputIteratorRA> +__device__ __forceinline__ void LoadStriped( + int linear_tid, ///< [in] A suitable 1D thread-identifier for the calling thread (e.g., (threadIdx.y * blockDim.x) + linear_tid for 2D thread blocks) + InputIteratorRA block_itr, ///< [in] The thread block's base input iterator for loading from + T (&items)[ITEMS_PER_THREAD]) ///< [out] Data to load +{ + #pragma unroll + for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ITEM++) + { + items[ITEM] = ThreadLoad(block_itr + (ITEM * BLOCK_THREADS) + linear_tid); + } +} + + +/** + * \brief Load a linear segment of items into a striped arrangement across the thread block using the specified cache modifier, guarded by range + * + * \striped + * + * \tparam MODIFIER cub::PtxLoadModifier cache modifier. + * \tparam BLOCK_THREADS The thread block size in threads + * \tparam T [inferred] The data type to load. + * \tparam ITEMS_PER_THREAD [inferred] The number of consecutive items partitioned onto each thread. + * \tparam InputIteratorRA [inferred] The random-access iterator type for input (may be a simple pointer type). + */ +template < + PtxLoadModifier MODIFIER, + int BLOCK_THREADS, + typename T, + int ITEMS_PER_THREAD, + typename InputIteratorRA> +__device__ __forceinline__ void LoadStriped( + int linear_tid, ///< [in] A suitable 1D thread-identifier for the calling thread (e.g., (threadIdx.y * blockDim.x) + linear_tid for 2D thread blocks) + InputIteratorRA block_itr, ///< [in] The thread block's base input iterator for loading from + T (&items)[ITEMS_PER_THREAD], ///< [out] Data to load + int valid_items) ///< [in] Number of valid items to load +{ + int bounds = valid_items - linear_tid; + + #pragma unroll + for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ITEM++) + { + if (ITEM * BLOCK_THREADS < bounds) + { + items[ITEM] = ThreadLoad(block_itr + linear_tid + (ITEM * BLOCK_THREADS)); + } + } +} + + +/** + * \brief Load a linear segment of items into a striped arrangement across the thread block using the specified cache modifier, guarded by range, with a fall-back assignment of out-of-bound elements. + * + * \striped + * + * \tparam MODIFIER cub::PtxLoadModifier cache modifier. + * \tparam BLOCK_THREADS The thread block size in threads + * \tparam T [inferred] The data type to load. + * \tparam ITEMS_PER_THREAD [inferred] The number of consecutive items partitioned onto each thread. + * \tparam InputIteratorRA [inferred] The random-access iterator type for input (may be a simple pointer type). + */ +template < + PtxLoadModifier MODIFIER, + int BLOCK_THREADS, + typename T, + int ITEMS_PER_THREAD, + typename InputIteratorRA> +__device__ __forceinline__ void LoadStriped( + int linear_tid, ///< [in] A suitable 1D thread-identifier for the calling thread (e.g., (threadIdx.y * blockDim.x) + linear_tid for 2D thread blocks) + InputIteratorRA block_itr, ///< [in] The thread block's base input iterator for loading from + T (&items)[ITEMS_PER_THREAD], ///< [out] Data to load + int valid_items, ///< [in] Number of valid items to load + T oob_default) ///< [in] Default value to assign out-of-bound items +{ + int bounds = valid_items - linear_tid; + + #pragma unroll + for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ITEM++) + { + items[ITEM] = (ITEM * BLOCK_THREADS < bounds) ? + ThreadLoad(block_itr + linear_tid + (ITEM * BLOCK_THREADS)) : + oob_default; + } +} + + + +//@} end member group +/******************************************************************//** + * \name Warp-striped I/O + *********************************************************************/ +//@{ + + +/** + * \brief Load a linear segment of items into a warp-striped arrangement across the thread block using the specified cache modifier. + * + * \warpstriped + * + * \par Usage Considerations + * The number of threads in the thread block must be a multiple of the architecture's warp size. + * + * \tparam MODIFIER cub::PtxLoadModifier cache modifier. + * \tparam T [inferred] The data type to load. + * \tparam ITEMS_PER_THREAD [inferred] The number of consecutive items partitioned onto each thread. + * \tparam InputIteratorRA [inferred] The random-access iterator type for input (may be a simple pointer type). + */ +template < + PtxLoadModifier MODIFIER, + typename T, + int ITEMS_PER_THREAD, + typename InputIteratorRA> +__device__ __forceinline__ void LoadWarpStriped( + int linear_tid, ///< [in] A suitable 1D thread-identifier for the calling thread (e.g., (threadIdx.y * blockDim.x) + linear_tid for 2D thread blocks) + InputIteratorRA block_itr, ///< [in] The thread block's base input iterator for loading from + T (&items)[ITEMS_PER_THREAD]) ///< [out] Data to load +{ + int tid = linear_tid & (PtxArchProps::WARP_THREADS - 1); + int wid = linear_tid >> PtxArchProps::LOG_WARP_THREADS; + int warp_offset = wid * PtxArchProps::WARP_THREADS * ITEMS_PER_THREAD; + + // Load directly in warp-striped order + #pragma unroll + for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ITEM++) + { + items[ITEM] = ThreadLoad(block_itr + warp_offset + tid + (ITEM * PtxArchProps::WARP_THREADS)); + } +} + + +/** + * \brief Load a linear segment of items into a warp-striped arrangement across the thread block using the specified cache modifier, guarded by range + * + * \warpstriped + * + * \par Usage Considerations + * The number of threads in the thread block must be a multiple of the architecture's warp size. + * + * \tparam MODIFIER cub::PtxLoadModifier cache modifier. + * \tparam T [inferred] The data type to load. + * \tparam ITEMS_PER_THREAD [inferred] The number of consecutive items partitioned onto each thread. + * \tparam InputIteratorRA [inferred] The random-access iterator type for input (may be a simple pointer type). + */ +template < + PtxLoadModifier MODIFIER, + typename T, + int ITEMS_PER_THREAD, + typename InputIteratorRA> +__device__ __forceinline__ void LoadWarpStriped( + int linear_tid, ///< [in] A suitable 1D thread-identifier for the calling thread (e.g., (threadIdx.y * blockDim.x) + linear_tid for 2D thread blocks) + InputIteratorRA block_itr, ///< [in] The thread block's base input iterator for loading from + T (&items)[ITEMS_PER_THREAD], ///< [out] Data to load + int valid_items) ///< [in] Number of valid items to load +{ + int tid = linear_tid & (PtxArchProps::WARP_THREADS - 1); + int wid = linear_tid >> PtxArchProps::LOG_WARP_THREADS; + int warp_offset = wid * PtxArchProps::WARP_THREADS * ITEMS_PER_THREAD; + int bounds = valid_items - warp_offset - tid; + + // Load directly in warp-striped order + #pragma unroll + for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ITEM++) + { + if ((ITEM * PtxArchProps::WARP_THREADS) < bounds) + { + items[ITEM] = ThreadLoad(block_itr + warp_offset + tid + (ITEM * PtxArchProps::WARP_THREADS)); + } + } +} + + +/** + * \brief Load a linear segment of items into a warp-striped arrangement across the thread block using the specified cache modifier, guarded by range, with a fall-back assignment of out-of-bound elements. + * + * \warpstriped + * + * \par Usage Considerations + * The number of threads in the thread block must be a multiple of the architecture's warp size. + * + * \tparam MODIFIER cub::PtxLoadModifier cache modifier. + * \tparam T [inferred] The data type to load. + * \tparam ITEMS_PER_THREAD [inferred] The number of consecutive items partitioned onto each thread. + * \tparam InputIteratorRA [inferred] The random-access iterator type for input (may be a simple pointer type). + */ +template < + PtxLoadModifier MODIFIER, + typename T, + int ITEMS_PER_THREAD, + typename InputIteratorRA> +__device__ __forceinline__ void LoadWarpStriped( + int linear_tid, ///< [in] A suitable 1D thread-identifier for the calling thread (e.g., (threadIdx.y * blockDim.x) + linear_tid for 2D thread blocks) + InputIteratorRA block_itr, ///< [in] The thread block's base input iterator for loading from + T (&items)[ITEMS_PER_THREAD], ///< [out] Data to load + int valid_items, ///< [in] Number of valid items to load + T oob_default) ///< [in] Default value to assign out-of-bound items +{ + int tid = linear_tid & (PtxArchProps::WARP_THREADS - 1); + int wid = linear_tid >> PtxArchProps::LOG_WARP_THREADS; + int warp_offset = wid * PtxArchProps::WARP_THREADS * ITEMS_PER_THREAD; + int bounds = valid_items - warp_offset - tid; + + // Load directly in warp-striped order + #pragma unroll + for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ITEM++) + { + items[ITEM] = ((ITEM * PtxArchProps::WARP_THREADS) < bounds) ? + ThreadLoad(block_itr + warp_offset + tid + (ITEM * PtxArchProps::WARP_THREADS)) : + oob_default; + } +} + + + +//@} end member group +/******************************************************************//** + * \name Blocked, vectorized I/O + *********************************************************************/ +//@{ + +/** + * \brief Load a linear segment of items into a blocked arrangement across the thread block using the specified cache modifier. + * + * \blocked + * + * The input offset (\p block_ptr + \p block_offset) must be quad-item aligned + * + * The following conditions will prevent vectorization and loading will fall back to cub::BLOCK_LOAD_DIRECT: + * - \p ITEMS_PER_THREAD is odd + * - The data type \p T is not a built-in primitive or CUDA vector type (e.g., \p short, \p int2, \p double, \p float2, etc.) + * + * \tparam MODIFIER cub::PtxLoadModifier cache modifier. + * \tparam T [inferred] The data type to load. + * \tparam ITEMS_PER_THREAD [inferred] The number of consecutive items partitioned onto each thread. + */ +template < + PtxLoadModifier MODIFIER, + typename T, + int ITEMS_PER_THREAD> +__device__ __forceinline__ void LoadBlockedVectorized( + int linear_tid, ///< [in] A suitable 1D thread-identifier for the calling thread (e.g., (threadIdx.y * blockDim.x) + linear_tid for 2D thread blocks) + T *block_ptr, ///< [in] Input pointer for loading from + T (&items)[ITEMS_PER_THREAD]) ///< [out] Data to load +{ + enum + { + // Maximum CUDA vector size is 4 elements + MAX_VEC_SIZE = CUB_MIN(4, ITEMS_PER_THREAD), + + // Vector size must be a power of two and an even divisor of the items per thread + VEC_SIZE = ((((MAX_VEC_SIZE - 1) & MAX_VEC_SIZE) == 0) && ((ITEMS_PER_THREAD % MAX_VEC_SIZE) == 0)) ? + MAX_VEC_SIZE : + 1, + + VECTORS_PER_THREAD = ITEMS_PER_THREAD / VEC_SIZE, + }; + + // Vector type + typedef typename VectorHelper::Type Vector; + + // Alias local data (use raw_items array here which should get optimized away to prevent conservative PTXAS lmem spilling) + T raw_items[ITEMS_PER_THREAD]; + + // Direct-load using vector types + LoadBlocked( + linear_tid, + reinterpret_cast(block_ptr), + reinterpret_cast(raw_items)); + + // Copy + #pragma unroll + for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ITEM++) + { + items[ITEM] = raw_items[ITEM]; + } +} + + +//@} end member group + +/** @} */ // end group IoModule + + + +//----------------------------------------------------------------------------- +// Generic BlockLoad abstraction +//----------------------------------------------------------------------------- + +/** + * \brief cub::BlockLoadAlgorithm enumerates alternative algorithms for cub::BlockLoad to read a linear segment of data from memory into a blocked arrangement across a CUDA thread block. + */ +enum BlockLoadAlgorithm +{ + /** + * \par Overview + * + * A [blocked arrangement](index.html#sec5sec4) of data is read + * directly from memory. The thread block reads items in a parallel "raking" fashion: threadi + * reads the ith segment of consecutive elements. + * + * \par Performance Considerations + * - The utilization of memory transactions (coalescing) decreases as the + * access stride between threads increases (i.e., the number items per thread). + */ + BLOCK_LOAD_DIRECT, + + /** + * \par Overview + * + * A [blocked arrangement](index.html#sec5sec4) of data is read directly + * from memory using CUDA's built-in vectorized loads as a coalescing optimization. + * The thread block reads items in a parallel "raking" fashion: threadi uses vector loads to + * read the ith segment of consecutive elements. + * + * For example, ld.global.v4.s32 instructions will be generated when \p T = \p int and \p ITEMS_PER_THREAD > 4. + * + * \par Performance Considerations + * - The utilization of memory transactions (coalescing) remains high until the the + * access stride between threads (i.e., the number items per thread) exceeds the + * maximum vector load width (typically 4 items or 64B, whichever is lower). + * - The following conditions will prevent vectorization and loading will fall back to cub::BLOCK_LOAD_DIRECT: + * - \p ITEMS_PER_THREAD is odd + * - The \p InputIteratorRA is not a simple pointer type + * - The block input offset is not quadword-aligned + * - The data type \p T is not a built-in primitive or CUDA vector type (e.g., \p short, \p int2, \p double, \p float2, etc.) + */ + BLOCK_LOAD_VECTORIZE, + + /** + * \par Overview + * + * A [striped arrangement](index.html#sec5sec4) of data is read + * directly from memory and then is locally transposed into a + * [blocked arrangement](index.html#sec5sec4). The thread block + * reads items in a parallel "strip-mining" fashion: + * threadi reads items having stride \p BLOCK_THREADS + * between them. cub::BlockExchange is then used to locally reorder the items + * into a [blocked arrangement](index.html#sec5sec4). + * + * \par Performance Considerations + * - The utilization of memory transactions (coalescing) remains high regardless + * of items loaded per thread. + * - The local reordering incurs slightly longer latencies and throughput than the + * direct cub::BLOCK_LOAD_DIRECT and cub::BLOCK_LOAD_VECTORIZE alternatives. + */ + BLOCK_LOAD_TRANSPOSE, + + + /** + * \par Overview + * + * A [warp-striped arrangement](index.html#sec5sec4) of data is read + * directly from memory and then is locally transposed into a + * [blocked arrangement](index.html#sec5sec4). Each warp reads its own + * contiguous segment in a parallel "strip-mining" fashion: lanei + * reads items having stride \p WARP_THREADS between them. cub::BlockExchange + * is then used to locally reorder the items into a + * [blocked arrangement](index.html#sec5sec4). + * + * \par Usage Considerations + * - BLOCK_THREADS must be a multiple of WARP_THREADS + * + * \par Performance Considerations + * - The utilization of memory transactions (coalescing) remains high regardless + * of items loaded per thread. + * - The local reordering incurs slightly longer latencies and throughput than the + * direct cub::BLOCK_LOAD_DIRECT and cub::BLOCK_LOAD_VECTORIZE alternatives. + */ + BLOCK_LOAD_WARP_TRANSPOSE, +}; + + +/** + * \brief The BlockLoad class provides [collective](index.html#sec0) data movement methods for loading a linear segment of items from memory into a [blocked arrangement](index.html#sec5sec4) across a CUDA thread block. ![](block_load_logo.png) + * \ingroup BlockModule + * + * \par Overview + * The BlockLoad class provides a single data movement abstraction that can be specialized + * to implement different cub::BlockLoadAlgorithm strategies. This facilitates different + * performance policies for different architectures, data types, granularity sizes, etc. + * + * \par + * Optionally, BlockLoad can be specialized by different data movement strategies: + * -# cub::BLOCK_LOAD_DIRECT. A [blocked arrangement](index.html#sec5sec4) + * of data is read directly from memory. [More...](\ref cub::BlockLoadAlgorithm) + * -# cub::BLOCK_LOAD_VECTORIZE. A [blocked arrangement](index.html#sec5sec4) + * of data is read directly from memory using CUDA's built-in vectorized loads as a + * coalescing optimization. [More...](\ref cub::BlockLoadAlgorithm) + * -# cub::BLOCK_LOAD_TRANSPOSE. A [striped arrangement](index.html#sec5sec4) + * of data is read directly from memory and is then locally transposed into a + * [blocked arrangement](index.html#sec5sec4). [More...](\ref cub::BlockLoadAlgorithm) + * -# cub::BLOCK_LOAD_WARP_TRANSPOSE. A [warp-striped arrangement](index.html#sec5sec4) + * of data is read directly from memory and is then locally transposed into a + * [blocked arrangement](index.html#sec5sec4). [More...](\ref cub::BlockLoadAlgorithm) + * + * \tparam InputIteratorRA The input iterator type (may be a simple pointer type). + * \tparam BLOCK_THREADS The thread block size in threads. + * \tparam ITEMS_PER_THREAD The number of consecutive items partitioned onto each thread. + * \tparam ALGORITHM [optional] cub::BlockLoadAlgorithm tuning policy. default: cub::BLOCK_LOAD_DIRECT. + * \tparam MODIFIER [optional] cub::PtxLoadModifier cache modifier. default: cub::LOAD_DEFAULT. + * \tparam WARP_TIME_SLICING [optional] For transposition-based cub::BlockLoadAlgorithm parameterizations that utilize shared memory: When \p true, only use enough shared memory for a single warp's worth of data, time-slicing the block-wide exchange over multiple synchronized rounds (default: false) + * + * \par A Simple Example + * \blockcollective{BlockLoad} + * \par + * The code snippet below illustrates the loading of a linear + * segment of 512 integers into a "blocked" arrangement across 128 threads where each + * thread owns 4 consecutive items. The load is specialized for \p BLOCK_LOAD_WARP_TRANSPOSE, + * meaning memory references are efficiently coalesced using a warp-striped access + * pattern (after which items are locally reordered among threads). + * \par + * \code + * #include + * + * __global__ void ExampleKernel(int *d_data, ...) + * { + * // Specialize BlockLoad for 128 threads owning 4 integer items each + * typedef cub::BlockLoad BlockLoad; + * + * // Allocate shared memory for BlockLoad + * __shared__ typename BlockLoad::TempStorage temp_storage; + * + * // Load a segment of consecutive items that are blocked across threads + * int thread_data[4]; + * BlockLoad(temp_storage).Load(d_data, thread_data); + * + * \endcode + * \par + * Suppose the input \p d_data is 0, 1, 2, 3, 4, 5, .... + * The set of \p thread_data across the block of threads in those threads will be + * { [0,1,2,3], [4,5,6,7], ..., [508,509,510,511] }. + * + */ +template < + typename InputIteratorRA, + int BLOCK_THREADS, + int ITEMS_PER_THREAD, + BlockLoadAlgorithm ALGORITHM = BLOCK_LOAD_DIRECT, + PtxLoadModifier MODIFIER = LOAD_DEFAULT, + bool WARP_TIME_SLICING = false> +class BlockLoad +{ +private: + + /****************************************************************************** + * Constants and typed definitions + ******************************************************************************/ + + // Data type of input iterator + typedef typename std::iterator_traits::value_type T; + + + /****************************************************************************** + * Algorithmic variants + ******************************************************************************/ + + /// Load helper + template + struct LoadInternal; + + + /** + * BLOCK_LOAD_DIRECT specialization of load helper + */ + template + struct LoadInternal + { + /// Shared memory storage layout type + typedef NullType TempStorage; + + /// Linear thread-id + int linear_tid; + + /// Constructor + __device__ __forceinline__ LoadInternal( + TempStorage &temp_storage, + int linear_tid) + : + linear_tid(linear_tid) + {} + + /// Load a linear segment of items from memory + __device__ __forceinline__ void Load( + InputIteratorRA block_itr, ///< [in] The thread block's base input iterator for loading from + T (&items)[ITEMS_PER_THREAD]) ///< [out] Data to load + { + LoadBlocked(linear_tid, block_itr, items); + } + + /// Load a linear segment of items from memory, guarded by range + __device__ __forceinline__ void Load( + InputIteratorRA block_itr, ///< [in] The thread block's base input iterator for loading from + T (&items)[ITEMS_PER_THREAD], ///< [out] Data to load + int valid_items) ///< [in] Number of valid items to load + { + LoadBlocked(linear_tid, block_itr, items, valid_items); + } + + /// Load a linear segment of items from memory, guarded by range, with a fall-back assignment of out-of-bound elements + __device__ __forceinline__ void Load( + InputIteratorRA block_itr, ///< [in] The thread block's base input iterator for loading from + T (&items)[ITEMS_PER_THREAD], ///< [out] Data to load + int valid_items, ///< [in] Number of valid items to load + T oob_default) ///< [in] Default value to assign out-of-bound items + { + LoadBlocked(linear_tid, block_itr, items, valid_items, oob_default); + } + + }; + + + /** + * BLOCK_LOAD_VECTORIZE specialization of load helper + */ + template + struct LoadInternal + { + /// Shared memory storage layout type + typedef NullType TempStorage; + + /// Linear thread-id + int linear_tid; + + /// Constructor + __device__ __forceinline__ LoadInternal( + TempStorage &temp_storage, + int linear_tid) + : + linear_tid(linear_tid) + {} + + /// Load a linear segment of items from memory, specialized for native pointer types (attempts vectorization) + __device__ __forceinline__ void Load( + T *block_ptr, ///< [in] The thread block's base input iterator for loading from + T (&items)[ITEMS_PER_THREAD]) ///< [out] Data to load + { + LoadBlockedVectorized(linear_tid, block_ptr, items); + } + + /// Load a linear segment of items from memory, specialized for opaque input iterators (skips vectorization) + template < + typename T, + typename _InputIteratorRA> + __device__ __forceinline__ void Load( + _InputIteratorRA block_itr, ///< [in] The thread block's base input iterator for loading from + T (&items)[ITEMS_PER_THREAD]) ///< [out] Data to load + { + LoadBlocked(linear_tid, block_itr, items); + } + + /// Load a linear segment of items from memory, guarded by range (skips vectorization) + __device__ __forceinline__ void Load( + InputIteratorRA block_itr, ///< [in] The thread block's base input iterator for loading from + T (&items)[ITEMS_PER_THREAD], ///< [out] Data to load + int valid_items) ///< [in] Number of valid items to load + { + LoadBlocked(linear_tid, block_itr, items, valid_items); + } + + /// Load a linear segment of items from memory, guarded by range, with a fall-back assignment of out-of-bound elements (skips vectorization) + __device__ __forceinline__ void Load( + InputIteratorRA block_itr, ///< [in] The thread block's base input iterator for loading from + T (&items)[ITEMS_PER_THREAD], ///< [out] Data to load + int valid_items, ///< [in] Number of valid items to load + T oob_default) ///< [in] Default value to assign out-of-bound items + { + LoadBlocked(linear_tid, block_itr, items, valid_items, oob_default); + } + + }; + + + /** + * BLOCK_LOAD_TRANSPOSE specialization of load helper + */ + template + struct LoadInternal + { + // BlockExchange utility type for keys + typedef BlockExchange BlockExchange; + + /// Shared memory storage layout type + typedef typename BlockExchange::TempStorage _TempStorage; + + /// Alias wrapper allowing storage to be unioned + struct TempStorage : Uninitialized<_TempStorage> {}; + + /// Thread reference to shared storage + _TempStorage &temp_storage; + + /// Linear thread-id + int linear_tid; + + /// Constructor + __device__ __forceinline__ LoadInternal( + TempStorage &temp_storage, + int linear_tid) + : + temp_storage(temp_storage.Alias()), + linear_tid(linear_tid) + {} + + /// Load a linear segment of items from memory + __device__ __forceinline__ void Load( + InputIteratorRA block_itr, ///< [in] The thread block's base input iterator for loading from + T (&items)[ITEMS_PER_THREAD]) ///< [out] Data to load{ + { + LoadStriped(linear_tid, block_itr, items); + BlockExchange(temp_storage, linear_tid).StripedToBlocked(items); + } + + /// Load a linear segment of items from memory, guarded by range + __device__ __forceinline__ void Load( + InputIteratorRA block_itr, ///< [in] The thread block's base input iterator for loading from + T (&items)[ITEMS_PER_THREAD], ///< [out] Data to load + int valid_items) ///< [in] Number of valid items to load + { + LoadStriped(linear_tid, block_itr, items, valid_items); + BlockExchange(temp_storage, linear_tid).StripedToBlocked(items); + } + + /// Load a linear segment of items from memory, guarded by range, with a fall-back assignment of out-of-bound elements + __device__ __forceinline__ void Load( + InputIteratorRA block_itr, ///< [in] The thread block's base input iterator for loading from + T (&items)[ITEMS_PER_THREAD], ///< [out] Data to load + int valid_items, ///< [in] Number of valid items to load + T oob_default) ///< [in] Default value to assign out-of-bound items + { + LoadStriped(linear_tid, block_itr, items, valid_items, oob_default); + BlockExchange(temp_storage, linear_tid).StripedToBlocked(items); + } + + }; + + + /** + * BLOCK_LOAD_WARP_TRANSPOSE specialization of load helper + */ + template + struct LoadInternal + { + enum + { + WARP_THREADS = PtxArchProps::WARP_THREADS + }; + + // Assert BLOCK_THREADS must be a multiple of WARP_THREADS + CUB_STATIC_ASSERT((BLOCK_THREADS % WARP_THREADS == 0), "BLOCK_THREADS must be a multiple of WARP_THREADS"); + + // BlockExchange utility type for keys + typedef BlockExchange BlockExchange; + + /// Shared memory storage layout type + typedef typename BlockExchange::TempStorage _TempStorage; + + /// Alias wrapper allowing storage to be unioned + struct TempStorage : Uninitialized<_TempStorage> {}; + + /// Thread reference to shared storage + _TempStorage &temp_storage; + + /// Linear thread-id + int linear_tid; + + /// Constructor + __device__ __forceinline__ LoadInternal( + TempStorage &temp_storage, + int linear_tid) + : + temp_storage(temp_storage.Alias()), + linear_tid(linear_tid) + {} + + /// Load a linear segment of items from memory + __device__ __forceinline__ void Load( + InputIteratorRA block_itr, ///< [in] The thread block's base input iterator for loading from + T (&items)[ITEMS_PER_THREAD]) ///< [out] Data to load{ + { + LoadWarpStriped(linear_tid, block_itr, items); + BlockExchange(temp_storage, linear_tid).WarpStripedToBlocked(items); + } + + /// Load a linear segment of items from memory, guarded by range + __device__ __forceinline__ void Load( + InputIteratorRA block_itr, ///< [in] The thread block's base input iterator for loading from + T (&items)[ITEMS_PER_THREAD], ///< [out] Data to load + int valid_items) ///< [in] Number of valid items to load + { + LoadWarpStriped(linear_tid, block_itr, items, valid_items); + BlockExchange(temp_storage, linear_tid).WarpStripedToBlocked(items); + } + + + /// Load a linear segment of items from memory, guarded by range, with a fall-back assignment of out-of-bound elements + __device__ __forceinline__ void Load( + InputIteratorRA block_itr, ///< [in] The thread block's base input iterator for loading from + T (&items)[ITEMS_PER_THREAD], ///< [out] Data to load + int valid_items, ///< [in] Number of valid items to load + T oob_default) ///< [in] Default value to assign out-of-bound items + { + LoadWarpStriped(linear_tid, block_itr, items, valid_items, oob_default); + BlockExchange(temp_storage, linear_tid).WarpStripedToBlocked(items); + } + }; + + + /****************************************************************************** + * Type definitions + ******************************************************************************/ + + /// Internal load implementation to use + typedef LoadInternal InternalLoad; + + + /// Shared memory storage layout type + typedef typename InternalLoad::TempStorage _TempStorage; + + + /****************************************************************************** + * Utility methods + ******************************************************************************/ + + /// Internal storage allocator + __device__ __forceinline__ _TempStorage& PrivateStorage() + { + __shared__ _TempStorage private_storage; + return private_storage; + } + + + /****************************************************************************** + * Thread fields + ******************************************************************************/ + + /// Thread reference to shared storage + _TempStorage &temp_storage; + + /// Linear thread-id + int linear_tid; + +public: + + /// \smemstorage{BlockLoad} + struct TempStorage : Uninitialized<_TempStorage> {}; + + + /******************************************************************//** + * \name Collective constructors + *********************************************************************/ + //@{ + + /** + * \brief Collective constructor for 1D thread blocks using a private static allocation of shared memory as temporary storage. Threads are identified using threadIdx.x. + */ + __device__ __forceinline__ BlockLoad() + : + temp_storage(PrivateStorage()), + linear_tid(threadIdx.x) + {} + + + /** + * \brief Collective constructor for 1D thread blocks using the specified memory allocation as temporary storage. Threads are identified using threadIdx.x. + */ + __device__ __forceinline__ BlockLoad( + TempStorage &temp_storage) ///< [in] Reference to memory allocation having layout type TempStorage + : + temp_storage(temp_storage.Alias()), + linear_tid(threadIdx.x) + {} + + + /** + * \brief Collective constructor using a private static allocation of shared memory as temporary storage. Each thread is identified using the supplied linear thread identifier + */ + __device__ __forceinline__ BlockLoad( + int linear_tid) ///< [in] A suitable 1D thread-identifier for the calling thread (e.g., (threadIdx.y * blockDim.x) + linear_tid for 2D thread blocks) + : + temp_storage(PrivateStorage()), + linear_tid(linear_tid) + {} + + + /** + * \brief Collective constructor using the specified memory allocation as temporary storage. Each thread is identified using the supplied linear thread identifier. + */ + __device__ __forceinline__ BlockLoad( + TempStorage &temp_storage, ///< [in] Reference to memory allocation having layout type TempStorage + int linear_tid) ///< [in] [optional] A suitable 1D thread-identifier for the calling thread (e.g., (threadIdx.y * blockDim.x) + linear_tid for 2D thread blocks) + : + temp_storage(temp_storage.Alias()), + linear_tid(linear_tid) + {} + + + + //@} end member group + /******************************************************************//** + * \name Data movement + *********************************************************************/ + //@{ + + + /** + * \brief Load a linear segment of items from memory. + * + * \blocked + * + * The code snippet below illustrates the loading of a linear + * segment of 512 integers into a "blocked" arrangement across 128 threads where each + * thread owns 4 consecutive items. The load is specialized for \p BLOCK_LOAD_WARP_TRANSPOSE, + * meaning memory references are efficiently coalesced using a warp-striped access + * pattern (after which items are locally reordered among threads). + * \par + * \code + * #include + * + * __global__ void ExampleKernel(int *d_data, ...) + * { + * // Specialize BlockLoad for 128 threads owning 4 integer items each + * typedef cub::BlockLoad BlockLoad; + * + * // Allocate shared memory for BlockLoad + * __shared__ typename BlockLoad::TempStorage temp_storage; + * + * // Load a segment of consecutive items that are blocked across threads + * int thread_data[4]; + * BlockLoad(temp_storage).Load(d_data, thread_data); + * + * \endcode + * \par + * Suppose the input \p d_data is 0, 1, 2, 3, 4, 5, .... + * The set of \p thread_data across the block of threads in those threads will be + * { [0,1,2,3], [4,5,6,7], ..., [508,509,510,511] }. + * + */ + __device__ __forceinline__ void Load( + InputIteratorRA block_itr, ///< [in] The thread block's base input iterator for loading from + T (&items)[ITEMS_PER_THREAD]) ///< [out] Data to load + { + InternalLoad(temp_storage, linear_tid).Load(block_itr, items); + } + + + /** + * \brief Load a linear segment of items from memory, guarded by range. + * + * \blocked + * + * The code snippet below illustrates the guarded loading of a linear + * segment of 512 integers into a "blocked" arrangement across 128 threads where each + * thread owns 4 consecutive items. The load is specialized for \p BLOCK_LOAD_WARP_TRANSPOSE, + * meaning memory references are efficiently coalesced using a warp-striped access + * pattern (after which items are locally reordered among threads). + * \par + * \code + * #include + * + * __global__ void ExampleKernel(int *d_data, int valid_items, ...) + * { + * // Specialize BlockLoad for 128 threads owning 4 integer items each + * typedef cub::BlockLoad BlockLoad; + * + * // Allocate shared memory for BlockLoad + * __shared__ typename BlockLoad::TempStorage temp_storage; + * + * // Load a segment of consecutive items that are blocked across threads + * int thread_data[4]; + * BlockLoad(temp_storage).Load(d_data, thread_data, valid_items); + * + * \endcode + * \par + * Suppose the input \p d_data is 0, 1, 2, 3, 4, 5, 6... and \p valid_items is \p 5. + * The set of \p thread_data across the block of threads in those threads will be + * { [0,1,2,3], [4,?,?,?], ..., [?,?,?,?] }, with only the first two threads + * being unmasked to load portions of valid data (and other items remaining unassigned). + * + */ + __device__ __forceinline__ void Load( + InputIteratorRA block_itr, ///< [in] The thread block's base input iterator for loading from + T (&items)[ITEMS_PER_THREAD], ///< [out] Data to load + int valid_items) ///< [in] Number of valid items to load + { + InternalLoad(temp_storage, linear_tid).Load(block_itr, items, valid_items); + } + + + /** + * \brief Load a linear segment of items from memory, guarded by range, with a fall-back assignment of out-of-bound elements + * + * \blocked + * + * The code snippet below illustrates the guarded loading of a linear + * segment of 512 integers into a "blocked" arrangement across 128 threads where each + * thread owns 4 consecutive items. The load is specialized for \p BLOCK_LOAD_WARP_TRANSPOSE, + * meaning memory references are efficiently coalesced using a warp-striped access + * pattern (after which items are locally reordered among threads). + * \par + * \code + * #include + * + * __global__ void ExampleKernel(int *d_data, int valid_items, ...) + * { + * // Specialize BlockLoad for 128 threads owning 4 integer items each + * typedef cub::BlockLoad BlockLoad; + * + * // Allocate shared memory for BlockLoad + * __shared__ typename BlockLoad::TempStorage temp_storage; + * + * // Load a segment of consecutive items that are blocked across threads + * int thread_data[4]; + * BlockLoad(temp_storage).Load(d_data, thread_data, valid_items, -1); + * + * \endcode + * \par + * Suppose the input \p d_data is 0, 1, 2, 3, 4, 5, 6..., + * \p valid_items is \p 5, and the out-of-bounds default is \p -1. + * The set of \p thread_data across the block of threads in those threads will be + * { [0,1,2,3], [4,-1,-1,-1], ..., [-1,-1,-1,-1] }, with only the first two threads + * being unmasked to load portions of valid data (and other items are assigned \p -1) + * + */ + __device__ __forceinline__ void Load( + InputIteratorRA block_itr, ///< [in] The thread block's base input iterator for loading from + T (&items)[ITEMS_PER_THREAD], ///< [out] Data to load + int valid_items, ///< [in] Number of valid items to load + T oob_default) ///< [in] Default value to assign out-of-bound items + { + InternalLoad(temp_storage, linear_tid).Load(block_itr, items, valid_items, oob_default); + } + + + //@} end member group + +}; + + +} // CUB namespace +CUB_NS_POSTFIX // Optional outer namespace(s) + diff --git a/lib/kokkos/TPL/cub/block/block_radix_rank.cuh b/lib/kokkos/TPL/cub/block/block_radix_rank.cuh new file mode 100755 index 0000000000..149a62c65f --- /dev/null +++ b/lib/kokkos/TPL/cub/block/block_radix_rank.cuh @@ -0,0 +1,479 @@ +/****************************************************************************** + * Copyright (c) 2011, Duane Merrill. All rights reserved. + * Copyright (c) 2011-2013, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the NVIDIA CORPORATION nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + ******************************************************************************/ + +/** + * \file + * cub::BlockRadixRank provides operations for ranking unsigned integer types within a CUDA threadblock + */ + +#pragma once + +#include "../util_arch.cuh" +#include "../util_type.cuh" +#include "../thread/thread_reduce.cuh" +#include "../thread/thread_scan.cuh" +#include "../block/block_scan.cuh" +#include "../util_namespace.cuh" + + +/// Optional outer namespace(s) +CUB_NS_PREFIX + +/// CUB namespace +namespace cub { + +/** + * \brief BlockRadixRank provides operations for ranking unsigned integer types within a CUDA threadblock. + * \ingroup BlockModule + * + * \par Overview + * Blah... + * + * \tparam BLOCK_THREADS The thread block size in threads + * \tparam RADIX_BITS [optional] The number of radix bits per digit place (default: 5 bits) + * \tparam MEMOIZE_OUTER_SCAN [optional] Whether or not to buffer outer raking scan partials to incur fewer shared memory reads at the expense of higher register pressure (default: true for architectures SM35 and newer, false otherwise). See BlockScanAlgorithm::BLOCK_SCAN_RAKING_MEMOIZE for more details. + * \tparam INNER_SCAN_ALGORITHM [optional] The cub::BlockScanAlgorithm algorithm to use (default: cub::BLOCK_SCAN_WARP_SCANS) + * \tparam SMEM_CONFIG [optional] Shared memory bank mode (default: \p cudaSharedMemBankSizeFourByte) + * + * \par Usage Considerations + * - Keys must be in a form suitable for radix ranking (i.e., unsigned bits). + * - Assumes a [blocked arrangement](index.html#sec5sec4) of elements across threads + * - \smemreuse{BlockRadixRank::TempStorage} + * + * \par Performance Considerations + * + * \par Algorithm + * These parallel radix ranking variants have O(n) work complexity and are implemented in XXX phases: + * -# blah + * -# blah + * + * \par Examples + * \par + * - Example 1: Simple radix rank of 32-bit integer keys + * \code + * #include + * + * template + * __global__ void ExampleKernel(...) + * { + * + * \endcode + */ +template < + int BLOCK_THREADS, + int RADIX_BITS, + bool MEMOIZE_OUTER_SCAN = (CUB_PTX_ARCH >= 350) ? true : false, + BlockScanAlgorithm INNER_SCAN_ALGORITHM = BLOCK_SCAN_WARP_SCANS, + cudaSharedMemConfig SMEM_CONFIG = cudaSharedMemBankSizeFourByte> +class BlockRadixRank +{ +private: + + /****************************************************************************** + * Type definitions and constants + ******************************************************************************/ + + // Integer type for digit counters (to be packed into words of type PackedCounters) + typedef unsigned short DigitCounter; + + // Integer type for packing DigitCounters into columns of shared memory banks + typedef typename If<(SMEM_CONFIG == cudaSharedMemBankSizeEightByte), + unsigned long long, + unsigned int>::Type PackedCounter; + + enum + { + RADIX_DIGITS = 1 << RADIX_BITS, + + LOG_WARP_THREADS = PtxArchProps::LOG_WARP_THREADS, + WARP_THREADS = 1 << LOG_WARP_THREADS, + WARPS = (BLOCK_THREADS + WARP_THREADS - 1) / WARP_THREADS, + + BYTES_PER_COUNTER = sizeof(DigitCounter), + LOG_BYTES_PER_COUNTER = Log2::VALUE, + + PACKING_RATIO = sizeof(PackedCounter) / sizeof(DigitCounter), + LOG_PACKING_RATIO = Log2::VALUE, + + LOG_COUNTER_LANES = CUB_MAX((RADIX_BITS - LOG_PACKING_RATIO), 0), // Always at least one lane + COUNTER_LANES = 1 << LOG_COUNTER_LANES, + + // The number of packed counters per thread (plus one for padding) + RAKING_SEGMENT = COUNTER_LANES + 1, + + LOG_SMEM_BANKS = PtxArchProps::LOG_SMEM_BANKS, + SMEM_BANKS = 1 << LOG_SMEM_BANKS, + }; + + + /// BlockScan type + typedef BlockScan BlockScan; + + + /// Shared memory storage layout type for BlockRadixRank + struct _TempStorage + { + // Storage for scanning local ranks + typename BlockScan::TempStorage block_scan; + + union + { + DigitCounter digit_counters[COUNTER_LANES + 1][BLOCK_THREADS][PACKING_RATIO]; + PackedCounter raking_grid[BLOCK_THREADS][RAKING_SEGMENT]; + }; + }; + + + /****************************************************************************** + * Thread fields + ******************************************************************************/ + + /// Shared storage reference + _TempStorage &temp_storage; + + /// Linear thread-id + int linear_tid; + + /// Copy of raking segment, promoted to registers + PackedCounter cached_segment[RAKING_SEGMENT]; + + + /****************************************************************************** + * Templated iteration + ******************************************************************************/ + + // General template iteration + template + struct Iterate + { + /** + * Decode keys. Decodes the radix digit from the current digit place + * and increments the thread's corresponding counter in shared + * memory for that digit. + * + * Saves both (1) the prior value of that counter (the key's + * thread-local exclusive prefix sum for that digit), and (2) the shared + * memory offset of the counter (for later use). + */ + template + static __device__ __forceinline__ void DecodeKeys( + BlockRadixRank &cta, // BlockRadixRank instance + UnsignedBits (&keys)[KEYS_PER_THREAD], // Key to decode + DigitCounter (&thread_prefixes)[KEYS_PER_THREAD], // Prefix counter value (out parameter) + DigitCounter* (&digit_counters)[KEYS_PER_THREAD], // Counter smem offset (out parameter) + int current_bit) // The least-significant bit position of the current digit to extract + { + // Add in sub-counter offset + UnsignedBits sub_counter = BFE(keys[COUNT], current_bit + LOG_COUNTER_LANES, LOG_PACKING_RATIO); + + // Add in row offset + UnsignedBits row_offset = BFE(keys[COUNT], current_bit, LOG_COUNTER_LANES); + + // Pointer to smem digit counter + digit_counters[COUNT] = &cta.temp_storage.digit_counters[row_offset][cta.linear_tid][sub_counter]; + + // Load thread-exclusive prefix + thread_prefixes[COUNT] = *digit_counters[COUNT]; + + // Store inclusive prefix + *digit_counters[COUNT] = thread_prefixes[COUNT] + 1; + + // Iterate next key + Iterate::DecodeKeys(cta, keys, thread_prefixes, digit_counters, current_bit); + } + + + // Termination + template + static __device__ __forceinline__ void UpdateRanks( + int (&ranks)[KEYS_PER_THREAD], // Local ranks (out parameter) + DigitCounter (&thread_prefixes)[KEYS_PER_THREAD], // Prefix counter value + DigitCounter* (&digit_counters)[KEYS_PER_THREAD]) // Counter smem offset + { + // Add in threadblock exclusive prefix + ranks[COUNT] = thread_prefixes[COUNT] + *digit_counters[COUNT]; + + // Iterate next key + Iterate::UpdateRanks(ranks, thread_prefixes, digit_counters); + } + }; + + + // Termination + template + struct Iterate + { + // DecodeKeys + template + static __device__ __forceinline__ void DecodeKeys( + BlockRadixRank &cta, + UnsignedBits (&keys)[KEYS_PER_THREAD], + DigitCounter (&thread_prefixes)[KEYS_PER_THREAD], + DigitCounter* (&digit_counters)[KEYS_PER_THREAD], + int current_bit) {} + + + // UpdateRanks + template + static __device__ __forceinline__ void UpdateRanks( + int (&ranks)[KEYS_PER_THREAD], + DigitCounter (&thread_prefixes)[KEYS_PER_THREAD], + DigitCounter *(&digit_counters)[KEYS_PER_THREAD]) {} + }; + + + /****************************************************************************** + * Utility methods + ******************************************************************************/ + + /** + * Internal storage allocator + */ + __device__ __forceinline__ _TempStorage& PrivateStorage() + { + __shared__ _TempStorage private_storage; + return private_storage; + } + + + /** + * Performs upsweep raking reduction, returning the aggregate + */ + __device__ __forceinline__ PackedCounter Upsweep() + { + PackedCounter *smem_raking_ptr = temp_storage.raking_grid[linear_tid]; + PackedCounter *raking_ptr; + + if (MEMOIZE_OUTER_SCAN) + { + // Copy data into registers + #pragma unroll + for (int i = 0; i < RAKING_SEGMENT; i++) + { + cached_segment[i] = smem_raking_ptr[i]; + } + raking_ptr = cached_segment; + } + else + { + raking_ptr = smem_raking_ptr; + } + + return ThreadReduce(raking_ptr, Sum()); + } + + + /// Performs exclusive downsweep raking scan + __device__ __forceinline__ void ExclusiveDownsweep( + PackedCounter raking_partial) + { + PackedCounter *smem_raking_ptr = temp_storage.raking_grid[linear_tid]; + + PackedCounter *raking_ptr = (MEMOIZE_OUTER_SCAN) ? + cached_segment : + smem_raking_ptr; + + // Exclusive raking downsweep scan + ThreadScanExclusive(raking_ptr, raking_ptr, Sum(), raking_partial); + + if (MEMOIZE_OUTER_SCAN) + { + // Copy data back to smem + #pragma unroll + for (int i = 0; i < RAKING_SEGMENT; i++) + { + smem_raking_ptr[i] = cached_segment[i]; + } + } + } + + + /** + * Reset shared memory digit counters + */ + __device__ __forceinline__ void ResetCounters() + { + // Reset shared memory digit counters + #pragma unroll + for (int LANE = 0; LANE < COUNTER_LANES + 1; LANE++) + { + *((PackedCounter*) temp_storage.digit_counters[LANE][linear_tid]) = 0; + } + } + + + /** + * Scan shared memory digit counters. + */ + __device__ __forceinline__ void ScanCounters() + { + // Upsweep scan + PackedCounter raking_partial = Upsweep(); + + // Compute inclusive sum + PackedCounter inclusive_partial; + PackedCounter packed_aggregate; + BlockScan(temp_storage.block_scan, linear_tid).InclusiveSum(raking_partial, inclusive_partial, packed_aggregate); + + // Propagate totals in packed fields + #pragma unroll + for (int PACKED = 1; PACKED < PACKING_RATIO; PACKED++) + { + inclusive_partial += packed_aggregate << (sizeof(DigitCounter) * 8 * PACKED); + } + + // Downsweep scan with exclusive partial + PackedCounter exclusive_partial = inclusive_partial - raking_partial; + ExclusiveDownsweep(exclusive_partial); + } + +public: + + /// \smemstorage{BlockScan} + struct TempStorage : Uninitialized<_TempStorage> {}; + + + /******************************************************************//** + * \name Collective constructors + *********************************************************************/ + //@{ + + /** + * \brief Collective constructor for 1D thread blocks using a private static allocation of shared memory as temporary storage. Threads are identified using threadIdx.x. + */ + __device__ __forceinline__ BlockRadixRank() + : + temp_storage(PrivateStorage()), + linear_tid(threadIdx.x) + {} + + + /** + * \brief Collective constructor for 1D thread blocks using the specified memory allocation as temporary storage. Threads are identified using threadIdx.x. + */ + __device__ __forceinline__ BlockRadixRank( + TempStorage &temp_storage) ///< [in] Reference to memory allocation having layout type TempStorage + : + temp_storage(temp_storage.Alias()), + linear_tid(threadIdx.x) + {} + + + /** + * \brief Collective constructor using a private static allocation of shared memory as temporary storage. Each thread is identified using the supplied linear thread identifier + */ + __device__ __forceinline__ BlockRadixRank( + int linear_tid) ///< [in] A suitable 1D thread-identifier for the calling thread (e.g., (threadIdx.y * blockDim.x) + linear_tid for 2D thread blocks) + : + temp_storage(PrivateStorage()), + linear_tid(linear_tid) + {} + + + /** + * \brief Collective constructor using the specified memory allocation as temporary storage. Each thread is identified using the supplied linear thread identifier. + */ + __device__ __forceinline__ BlockRadixRank( + TempStorage &temp_storage, ///< [in] Reference to memory allocation having layout type TempStorage + int linear_tid) ///< [in] [optional] A suitable 1D thread-identifier for the calling thread (e.g., (threadIdx.y * blockDim.x) + linear_tid for 2D thread blocks) + : + temp_storage(temp_storage.Alias()), + linear_tid(linear_tid) + {} + + + + //@} end member group + /******************************************************************//** + * \name Raking + *********************************************************************/ + //@{ + + /** + * \brief Rank keys. + */ + template < + typename UnsignedBits, + int KEYS_PER_THREAD> + __device__ __forceinline__ void RankKeys( + UnsignedBits (&keys)[KEYS_PER_THREAD], ///< [in] Keys for this tile + int (&ranks)[KEYS_PER_THREAD], ///< [out] For each key, the local rank within the tile + int current_bit) ///< [in] The least-significant bit position of the current digit to extract + { + DigitCounter thread_prefixes[KEYS_PER_THREAD]; // For each key, the count of previous keys in this tile having the same digit + DigitCounter* digit_counters[KEYS_PER_THREAD]; // For each key, the byte-offset of its corresponding digit counter in smem + + // Reset shared memory digit counters + ResetCounters(); + + // Decode keys and update digit counters + Iterate<0, KEYS_PER_THREAD>::DecodeKeys(*this, keys, thread_prefixes, digit_counters, current_bit); + + __syncthreads(); + + // Scan shared memory counters + ScanCounters(); + + __syncthreads(); + + // Extract the local ranks of each key + Iterate<0, KEYS_PER_THREAD>::UpdateRanks(ranks, thread_prefixes, digit_counters); + } + + + /** + * \brief Rank keys. For the lower \p RADIX_DIGITS threads, digit counts for each digit are provided for the corresponding thread. + */ + template < + typename UnsignedBits, + int KEYS_PER_THREAD> + __device__ __forceinline__ void RankKeys( + UnsignedBits (&keys)[KEYS_PER_THREAD], ///< [in] Keys for this tile + int (&ranks)[KEYS_PER_THREAD], ///< [out] For each key, the local rank within the tile (out parameter) + int current_bit, ///< [in] The least-significant bit position of the current digit to extract + int &inclusive_digit_prefix) ///< [out] The incluisve prefix sum for the digit threadIdx.x + { + // Rank keys + RankKeys(keys, ranks, current_bit); + + // Get the inclusive and exclusive digit totals corresponding to the calling thread. + if ((BLOCK_THREADS == RADIX_DIGITS) || (linear_tid < RADIX_DIGITS)) + { + // Obtain ex/inclusive digit counts. (Unfortunately these all reside in the + // first counter column, resulting in unavoidable bank conflicts.) + int counter_lane = (linear_tid & (COUNTER_LANES - 1)); + int sub_counter = linear_tid >> (LOG_COUNTER_LANES); + inclusive_digit_prefix = temp_storage.digit_counters[counter_lane + 1][0][sub_counter]; + } + } +}; + +} // CUB namespace +CUB_NS_POSTFIX // Optional outer namespace(s) + + diff --git a/lib/kokkos/TPL/cub/block/block_radix_sort.cuh b/lib/kokkos/TPL/cub/block/block_radix_sort.cuh new file mode 100755 index 0000000000..873d401266 --- /dev/null +++ b/lib/kokkos/TPL/cub/block/block_radix_sort.cuh @@ -0,0 +1,608 @@ +/****************************************************************************** + * Copyright (c) 2011, Duane Merrill. All rights reserved. + * Copyright (c) 2011-2013, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the NVIDIA CORPORATION nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + ******************************************************************************/ + +/** + * \file + * The cub::BlockRadixSort class provides [collective](index.html#sec0) methods for radix sorting of items partitioned across a CUDA thread block. + */ + + +#pragma once + +#include "../util_namespace.cuh" +#include "../util_arch.cuh" +#include "../util_type.cuh" +#include "block_exchange.cuh" +#include "block_radix_rank.cuh" + +/// Optional outer namespace(s) +CUB_NS_PREFIX + +/// CUB namespace +namespace cub { + +/** + * \brief The cub::BlockRadixSort class provides [collective](index.html#sec0) methods for sorting items partitioned across a CUDA thread block using a radix sorting method. ![](sorting_logo.png) + * \ingroup BlockModule + * + * \par Overview + * The [radix sorting method](http://en.wikipedia.org/wiki/Radix_sort) arranges + * items into ascending order. It relies upon a positional representation for + * keys, i.e., each key is comprised of an ordered sequence of symbols (e.g., digits, + * characters, etc.) specified from least-significant to most-significant. For a + * given input sequence of keys and a set of rules specifying a total ordering + * of the symbolic alphabet, the radix sorting method produces a lexicographic + * ordering of those keys. + * + * \par + * BlockRadixSort can sort all of the built-in C++ numeric primitive types, e.g.: + * unsigned char, \p int, \p double, etc. Within each key, the implementation treats fixed-length + * bit-sequences of \p RADIX_BITS as radix digit places. Although the direct radix sorting + * method can only be applied to unsigned integral types, BlockRadixSort + * is able to sort signed and floating-point types via simple bit-wise transformations + * that ensure lexicographic key ordering. + * + * \tparam Key Key type + * \tparam BLOCK_THREADS The thread block size in threads + * \tparam ITEMS_PER_THREAD The number of items per thread + * \tparam Value [optional] Value type (default: cub::NullType) + * \tparam RADIX_BITS [optional] The number of radix bits per digit place (default: 4 bits) + * \tparam MEMOIZE_OUTER_SCAN [optional] Whether or not to buffer outer raking scan partials to incur fewer shared memory reads at the expense of higher register pressure (default: true for architectures SM35 and newer, false otherwise). + * \tparam INNER_SCAN_ALGORITHM [optional] The cub::BlockScanAlgorithm algorithm to use (default: cub::BLOCK_SCAN_WARP_SCANS) + * \tparam SMEM_CONFIG [optional] Shared memory bank mode (default: \p cudaSharedMemBankSizeFourByte) + * + * \par A Simple Example + * \blockcollective{BlockRadixSort} + * \par + * The code snippet below illustrates a sort of 512 integer keys that + * are partitioned in a [blocked arrangement](index.html#sec5sec4) across 128 threads + * where each thread owns 4 consecutive items. + * \par + * \code + * #include + * + * __global__ void ExampleKernel(...) + * { + * // Specialize BlockRadixSort for 128 threads owning 4 integer items each + * typedef cub::BlockRadixSort BlockRadixSort; + * + * // Allocate shared memory for BlockRadixSort + * __shared__ typename BlockRadixSort::TempStorage temp_storage; + * + * // Obtain a segment of consecutive items that are blocked across threads + * int thread_keys[4]; + * ... + * + * // Collectively sort the keys + * BlockRadixSort(temp_storage).Sort(thread_keys); + * + * ... + * \endcode + * \par + * Suppose the set of input \p thread_keys across the block of threads is + * { [0,511,1,510], [2,509,3,508], [4,507,5,506], ..., [254,257,255,256] }. The + * corresponding output \p thread_keys in those threads will be + * { [0,1,2,3], [4,5,6,7], [8,9,10,11], ..., [508,509,510,511] }. + * + */ +template < + typename Key, + int BLOCK_THREADS, + int ITEMS_PER_THREAD, + typename Value = NullType, + int RADIX_BITS = 4, + bool MEMOIZE_OUTER_SCAN = (CUB_PTX_ARCH >= 350) ? true : false, + BlockScanAlgorithm INNER_SCAN_ALGORITHM = BLOCK_SCAN_WARP_SCANS, + cudaSharedMemConfig SMEM_CONFIG = cudaSharedMemBankSizeFourByte> +class BlockRadixSort +{ +private: + + /****************************************************************************** + * Constants and type definitions + ******************************************************************************/ + + // Key traits and unsigned bits type + typedef NumericTraits KeyTraits; + typedef typename KeyTraits::UnsignedBits UnsignedBits; + + /// BlockRadixRank utility type + typedef BlockRadixRank BlockRadixRank; + + /// BlockExchange utility type for keys + typedef BlockExchange BlockExchangeKeys; + + /// BlockExchange utility type for values + typedef BlockExchange BlockExchangeValues; + + /// Shared memory storage layout type + struct _TempStorage + { + union + { + typename BlockRadixRank::TempStorage ranking_storage; + typename BlockExchangeKeys::TempStorage exchange_keys; + typename BlockExchangeValues::TempStorage exchange_values; + }; + }; + + /****************************************************************************** + * Utility methods + ******************************************************************************/ + + /// Internal storage allocator + __device__ __forceinline__ _TempStorage& PrivateStorage() + { + __shared__ _TempStorage private_storage; + return private_storage; + } + + + /****************************************************************************** + * Thread fields + ******************************************************************************/ + + /// Shared storage reference + _TempStorage &temp_storage; + + /// Linear thread-id + int linear_tid; + + +public: + + /// \smemstorage{BlockScan} + struct TempStorage : Uninitialized<_TempStorage> {}; + + + /******************************************************************//** + * \name Collective constructors + *********************************************************************/ + //@{ + + /** + * \brief Collective constructor for 1D thread blocks using a private static allocation of shared memory as temporary storage. Threads are identified using threadIdx.x. + */ + __device__ __forceinline__ BlockRadixSort() + : + temp_storage(PrivateStorage()), + linear_tid(threadIdx.x) + {} + + + /** + * \brief Collective constructor for 1D thread blocks using the specified memory allocation as temporary storage. Threads are identified using threadIdx.x. + */ + __device__ __forceinline__ BlockRadixSort( + TempStorage &temp_storage) ///< [in] Reference to memory allocation having layout type TempStorage + : + temp_storage(temp_storage.Alias()), + linear_tid(threadIdx.x) + {} + + + /** + * \brief Collective constructor using a private static allocation of shared memory as temporary storage. Each thread is identified using the supplied linear thread identifier + */ + __device__ __forceinline__ BlockRadixSort( + int linear_tid) ///< [in] A suitable 1D thread-identifier for the calling thread (e.g., (threadIdx.y * blockDim.x) + linear_tid for 2D thread blocks) + : + temp_storage(PrivateStorage()), + linear_tid(linear_tid) + {} + + + /** + * \brief Collective constructor using the specified memory allocation as temporary storage. Each thread is identified using the supplied linear thread identifier. + */ + __device__ __forceinline__ BlockRadixSort( + TempStorage &temp_storage, ///< [in] Reference to memory allocation having layout type TempStorage + int linear_tid) ///< [in] [optional] A suitable 1D thread-identifier for the calling thread (e.g., (threadIdx.y * blockDim.x) + linear_tid for 2D thread blocks) + : + temp_storage(temp_storage.Alias()), + linear_tid(linear_tid) + {} + + + + //@} end member group + /******************************************************************//** + * \name Sorting (blocked arrangements) + *********************************************************************/ + //@{ + + /** + * \brief Performs a block-wide radix sort over a [blocked arrangement](index.html#sec5sec4) of keys. + * + * \smemreuse + * + * The code snippet below illustrates a sort of 512 integer keys that + * are partitioned in a [blocked arrangement](index.html#sec5sec4) across 128 threads + * where each thread owns 4 consecutive keys. + * \par + * \code + * #include + * + * __global__ void ExampleKernel(...) + * { + * // Specialize BlockRadixSort for 128 threads owning 4 integer keys each + * typedef cub::BlockRadixSort BlockRadixSort; + * + * // Allocate shared memory for BlockRadixSort + * __shared__ typename BlockRadixSort::TempStorage temp_storage; + * + * // Obtain a segment of consecutive items that are blocked across threads + * int thread_keys[4]; + * ... + * + * // Collectively sort the keys + * BlockRadixSort(temp_storage).Sort(thread_keys); + * + * \endcode + * \par + * Suppose the set of input \p thread_keys across the block of threads is + * { [0,511,1,510], [2,509,3,508], [4,507,5,506], ..., [254,257,255,256] }. + * The corresponding output \p thread_keys in those threads will be + * { [0,1,2,3], [4,5,6,7], [8,9,10,11], ..., [508,509,510,511] }. + */ + __device__ __forceinline__ void Sort( + Key (&keys)[ITEMS_PER_THREAD], ///< [in-out] Keys to sort + int begin_bit = 0, ///< [in] [optional] The beginning (least-significant) bit index needed for key comparison + int end_bit = sizeof(Key) * 8) ///< [in] [optional] The past-the-end (most-significant) bit index needed for key comparison + { + UnsignedBits (&unsigned_keys)[ITEMS_PER_THREAD] = + reinterpret_cast(keys); + + // Twiddle bits if necessary + #pragma unroll + for (int KEY = 0; KEY < ITEMS_PER_THREAD; KEY++) + { + unsigned_keys[KEY] = KeyTraits::TwiddleIn(unsigned_keys[KEY]); + } + + // Radix sorting passes + while (true) + { + // Rank the blocked keys + int ranks[ITEMS_PER_THREAD]; + BlockRadixRank(temp_storage.ranking_storage, linear_tid).RankKeys(unsigned_keys, ranks, begin_bit); + begin_bit += RADIX_BITS; + + __syncthreads(); + + // Exchange keys through shared memory in blocked arrangement + BlockExchangeKeys(temp_storage.exchange_keys, linear_tid).ScatterToBlocked(keys, ranks); + + // Quit if done + if (begin_bit >= end_bit) break; + + __syncthreads(); + } + + // Untwiddle bits if necessary + #pragma unroll + for (int KEY = 0; KEY < ITEMS_PER_THREAD; KEY++) + { + unsigned_keys[KEY] = KeyTraits::TwiddleOut(unsigned_keys[KEY]); + } + } + + + /** + * \brief Performs a block-wide radix sort across a [blocked arrangement](index.html#sec5sec4) of keys and values. + * + * BlockRadixSort can only accommodate one associated tile of values. To "truck along" + * more than one tile of values, simply perform a key-value sort of the keys paired + * with a temporary value array that enumerates the key indices. The reordered indices + * can then be used as a gather-vector for exchanging other associated tile data through + * shared memory. + * + * \smemreuse + * + * The code snippet below illustrates a sort of 512 integer keys and values that + * are partitioned in a [blocked arrangement](index.html#sec5sec4) across 128 threads + * where each thread owns 4 consecutive pairs. + * \par + * \code + * #include + * + * __global__ void ExampleKernel(...) + * { + * // Specialize BlockRadixSort for 128 threads owning 4 integer keys and values each + * typedef cub::BlockRadixSort BlockRadixSort; + * + * // Allocate shared memory for BlockRadixSort + * __shared__ typename BlockRadixSort::TempStorage temp_storage; + * + * // Obtain a segment of consecutive items that are blocked across threads + * int thread_keys[4]; + * int thread_values[4]; + * ... + * + * // Collectively sort the keys and values among block threads + * BlockRadixSort(temp_storage).Sort(thread_keys, thread_values); + * + * \endcode + * \par + * Suppose the set of input \p thread_keys across the block of threads is + * { [0,511,1,510], [2,509,3,508], [4,507,5,506], ..., [254,257,255,256] }. The + * corresponding output \p thread_keys in those threads will be + * { [0,1,2,3], [4,5,6,7], [8,9,10,11], ..., [508,509,510,511] }. + * + */ + __device__ __forceinline__ void Sort( + Key (&keys)[ITEMS_PER_THREAD], ///< [in-out] Keys to sort + Value (&values)[ITEMS_PER_THREAD], ///< [in-out] Values to sort + int begin_bit = 0, ///< [in] [optional] The beginning (least-significant) bit index needed for key comparison + int end_bit = sizeof(Key) * 8) ///< [in] [optional] The past-the-end (most-significant) bit index needed for key comparison + { + UnsignedBits (&unsigned_keys)[ITEMS_PER_THREAD] = + reinterpret_cast(keys); + + // Twiddle bits if necessary + #pragma unroll + for (int KEY = 0; KEY < ITEMS_PER_THREAD; KEY++) + { + unsigned_keys[KEY] = KeyTraits::TwiddleIn(unsigned_keys[KEY]); + } + + // Radix sorting passes + while (true) + { + // Rank the blocked keys + int ranks[ITEMS_PER_THREAD]; + BlockRadixRank(temp_storage.ranking_storage, linear_tid).RankKeys(unsigned_keys, ranks, begin_bit); + begin_bit += RADIX_BITS; + + __syncthreads(); + + // Exchange keys through shared memory in blocked arrangement + BlockExchangeKeys(temp_storage.exchange_keys, linear_tid).ScatterToBlocked(keys, ranks); + + __syncthreads(); + + // Exchange values through shared memory in blocked arrangement + BlockExchangeValues(temp_storage.exchange_values, linear_tid).ScatterToBlocked(values, ranks); + + // Quit if done + if (begin_bit >= end_bit) break; + + __syncthreads(); + } + + // Untwiddle bits if necessary + #pragma unroll + for (int KEY = 0; KEY < ITEMS_PER_THREAD; KEY++) + { + unsigned_keys[KEY] = KeyTraits::TwiddleOut(unsigned_keys[KEY]); + } + } + + + //@} end member group + /******************************************************************//** + * \name Sorting (blocked arrangement -> striped arrangement) + *********************************************************************/ + //@{ + + + /** + * \brief Performs a radix sort across a [blocked arrangement](index.html#sec5sec4) of keys, leaving them in a [striped arrangement](index.html#sec5sec4). + * + * \smemreuse + * + * The code snippet below illustrates a sort of 512 integer keys that + * are initially partitioned in a [blocked arrangement](index.html#sec5sec4) across 128 threads + * where each thread owns 4 consecutive keys. The final partitioning is striped. + * \par + * \code + * #include + * + * __global__ void ExampleKernel(...) + * { + * // Specialize BlockRadixSort for 128 threads owning 4 integer keys each + * typedef cub::BlockRadixSort BlockRadixSort; + * + * // Allocate shared memory for BlockRadixSort + * __shared__ typename BlockRadixSort::TempStorage temp_storage; + * + * // Obtain a segment of consecutive items that are blocked across threads + * int thread_keys[4]; + * ... + * + * // Collectively sort the keys + * BlockRadixSort(temp_storage).SortBlockedToStriped(thread_keys); + * + * \endcode + * \par + * Suppose the set of input \p thread_keys across the block of threads is + * { [0,511,1,510], [2,509,3,508], [4,507,5,506], ..., [254,257,255,256] }. The + * corresponding output \p thread_keys in those threads will be + * { [0,128,256,384], [1,129,257,385], [2,130,258,386], ..., [127,255,383,511] }. + * + */ + __device__ __forceinline__ void SortBlockedToStriped( + Key (&keys)[ITEMS_PER_THREAD], ///< [in-out] Keys to sort + int begin_bit = 0, ///< [in] [optional] The beginning (least-significant) bit index needed for key comparison + int end_bit = sizeof(Key) * 8) ///< [in] [optional] The past-the-end (most-significant) bit index needed for key comparison + { + UnsignedBits (&unsigned_keys)[ITEMS_PER_THREAD] = + reinterpret_cast(keys); + + // Twiddle bits if necessary + #pragma unroll + for (int KEY = 0; KEY < ITEMS_PER_THREAD; KEY++) + { + unsigned_keys[KEY] = KeyTraits::TwiddleIn(unsigned_keys[KEY]); + } + + // Radix sorting passes + while (true) + { + // Rank the blocked keys + int ranks[ITEMS_PER_THREAD]; + BlockRadixRank(temp_storage.ranking_storage, linear_tid).RankKeys(unsigned_keys, ranks, begin_bit); + begin_bit += RADIX_BITS; + + __syncthreads(); + + // Check if this is the last pass + if (begin_bit >= end_bit) + { + // Last pass exchanges keys through shared memory in striped arrangement + BlockExchangeKeys(temp_storage.exchange_keys, linear_tid).ScatterToStriped(keys, ranks); + + // Quit + break; + } + + // Exchange keys through shared memory in blocked arrangement + BlockExchangeKeys(temp_storage.exchange_keys, linear_tid).ScatterToBlocked(keys, ranks); + + __syncthreads(); + } + + // Untwiddle bits if necessary + #pragma unroll + for (int KEY = 0; KEY < ITEMS_PER_THREAD; KEY++) + { + unsigned_keys[KEY] = KeyTraits::TwiddleOut(unsigned_keys[KEY]); + } + } + + + /** + * \brief Performs a radix sort across a [blocked arrangement](index.html#sec5sec4) of keys and values, leaving them in a [striped arrangement](index.html#sec5sec4). + * + * BlockRadixSort can only accommodate one associated tile of values. To "truck along" + * more than one tile of values, simply perform a key-value sort of the keys paired + * with a temporary value array that enumerates the key indices. The reordered indices + * can then be used as a gather-vector for exchanging other associated tile data through + * shared memory. + * + * \smemreuse + * + * The code snippet below illustrates a sort of 512 integer keys and values that + * are initially partitioned in a [blocked arrangement](index.html#sec5sec4) across 128 threads + * where each thread owns 4 consecutive pairs. The final partitioning is striped. + * \par + * \code + * #include + * + * __global__ void ExampleKernel(...) + * { + * // Specialize BlockRadixSort for 128 threads owning 4 integer keys and values each + * typedef cub::BlockRadixSort BlockRadixSort; + * + * // Allocate shared memory for BlockRadixSort + * __shared__ typename BlockRadixSort::TempStorage temp_storage; + * + * // Obtain a segment of consecutive items that are blocked across threads + * int thread_keys[4]; + * int thread_values[4]; + * ... + * + * // Collectively sort the keys and values among block threads + * BlockRadixSort(temp_storage).SortBlockedToStriped(thread_keys, thread_values); + * + * \endcode + * \par + * Suppose the set of input \p thread_keys across the block of threads is + * { [0,511,1,510], [2,509,3,508], [4,507,5,506], ..., [254,257,255,256] }. The + * corresponding output \p thread_keys in those threads will be + * { [0,128,256,384], [1,129,257,385], [2,130,258,386], ..., [127,255,383,511] }. + * + */ + __device__ __forceinline__ void SortBlockedToStriped( + Key (&keys)[ITEMS_PER_THREAD], ///< [in-out] Keys to sort + Value (&values)[ITEMS_PER_THREAD], ///< [in-out] Values to sort + int begin_bit = 0, ///< [in] [optional] The beginning (least-significant) bit index needed for key comparison + int end_bit = sizeof(Key) * 8) ///< [in] [optional] The past-the-end (most-significant) bit index needed for key comparison + { + UnsignedBits (&unsigned_keys)[ITEMS_PER_THREAD] = + reinterpret_cast(keys); + + // Twiddle bits if necessary + #pragma unroll + for (int KEY = 0; KEY < ITEMS_PER_THREAD; KEY++) + { + unsigned_keys[KEY] = KeyTraits::TwiddleIn(unsigned_keys[KEY]); + } + + // Radix sorting passes + while (true) + { + // Rank the blocked keys + int ranks[ITEMS_PER_THREAD]; + BlockRadixRank(temp_storage.ranking_storage, linear_tid).RankKeys(unsigned_keys, ranks, begin_bit); + begin_bit += RADIX_BITS; + + __syncthreads(); + + // Check if this is the last pass + if (begin_bit >= end_bit) + { + // Last pass exchanges keys through shared memory in striped arrangement + BlockExchangeKeys(temp_storage.exchange_keys, linear_tid).ScatterToStriped(keys, ranks); + + __syncthreads(); + + // Last pass exchanges through shared memory in striped arrangement + BlockExchangeValues(temp_storage.exchange_values, linear_tid).ScatterToStriped(values, ranks); + + // Quit + break; + } + + // Exchange keys through shared memory in blocked arrangement + BlockExchangeKeys(temp_storage.exchange_keys, linear_tid).ScatterToBlocked(keys, ranks); + + __syncthreads(); + + // Exchange values through shared memory in blocked arrangement + BlockExchangeValues(temp_storage.exchange_values, linear_tid).ScatterToBlocked(values, ranks); + + __syncthreads(); + } + + // Untwiddle bits if necessary + #pragma unroll + for (int KEY = 0; KEY < ITEMS_PER_THREAD; KEY++) + { + unsigned_keys[KEY] = KeyTraits::TwiddleOut(unsigned_keys[KEY]); + } + } + + + //@} end member group + +}; + +} // CUB namespace +CUB_NS_POSTFIX // Optional outer namespace(s) + diff --git a/lib/kokkos/TPL/cub/block/block_raking_layout.cuh b/lib/kokkos/TPL/cub/block/block_raking_layout.cuh new file mode 100755 index 0000000000..878a786cd9 --- /dev/null +++ b/lib/kokkos/TPL/cub/block/block_raking_layout.cuh @@ -0,0 +1,145 @@ +/****************************************************************************** + * Copyright (c) 2011, Duane Merrill. All rights reserved. + * Copyright (c) 2011-2013, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the NVIDIA CORPORATION nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + ******************************************************************************/ + +/** + * \file + * cub::BlockRakingLayout provides a conflict-free shared memory layout abstraction for warp-raking across thread block data. + */ + + +#pragma once + +#include "../util_macro.cuh" +#include "../util_arch.cuh" +#include "../util_namespace.cuh" + +/// Optional outer namespace(s) +CUB_NS_PREFIX + +/// CUB namespace +namespace cub { + +/** + * \brief BlockRakingLayout provides a conflict-free shared memory layout abstraction for raking across thread block data. ![](raking.png) + * \ingroup BlockModule + * + * \par Overview + * This type facilitates a shared memory usage pattern where a block of CUDA + * threads places elements into shared memory and then reduces the active + * parallelism to one "raking" warp of threads for serially aggregating consecutive + * sequences of shared items. Padding is inserted to eliminate bank conflicts + * (for most data types). + * + * \tparam T The data type to be exchanged. + * \tparam BLOCK_THREADS The thread block size in threads. + * \tparam BLOCK_STRIPS When strip-mining, the number of threadblock-strips per tile + */ +template < + typename T, + int BLOCK_THREADS, + int BLOCK_STRIPS = 1> +struct BlockRakingLayout +{ + //--------------------------------------------------------------------- + // Constants and typedefs + //--------------------------------------------------------------------- + + enum + { + /// The total number of elements that need to be cooperatively reduced + SHARED_ELEMENTS = + BLOCK_THREADS * BLOCK_STRIPS, + + /// Maximum number of warp-synchronous raking threads + MAX_RAKING_THREADS = + CUB_MIN(BLOCK_THREADS, PtxArchProps::WARP_THREADS), + + /// Number of raking elements per warp-synchronous raking thread (rounded up) + SEGMENT_LENGTH = + (SHARED_ELEMENTS + MAX_RAKING_THREADS - 1) / MAX_RAKING_THREADS, + + /// Never use a raking thread that will have no valid data (e.g., when BLOCK_THREADS is 62 and SEGMENT_LENGTH is 2, we should only use 31 raking threads) + RAKING_THREADS = + (SHARED_ELEMENTS + SEGMENT_LENGTH - 1) / SEGMENT_LENGTH, + + /// Pad each segment length with one element if it evenly divides the number of banks + SEGMENT_PADDING = + (PtxArchProps::SMEM_BANKS % SEGMENT_LENGTH == 0) ? 1 : 0, + + /// Total number of elements in the raking grid + GRID_ELEMENTS = + RAKING_THREADS * (SEGMENT_LENGTH + SEGMENT_PADDING), + + /// Whether or not we need bounds checking during raking (the number of reduction elements is not a multiple of the warp size) + UNGUARDED = + (SHARED_ELEMENTS % RAKING_THREADS == 0), + }; + + + /** + * \brief Shared memory storage type + */ + typedef T TempStorage[BlockRakingLayout::GRID_ELEMENTS]; + + + /** + * \brief Returns the location for the calling thread to place data into the grid + */ + static __device__ __forceinline__ T* PlacementPtr( + TempStorage &temp_storage, + int linear_tid, + int block_strip = 0) + { + // Offset for partial + unsigned int offset = (block_strip * BLOCK_THREADS) + linear_tid; + + // Add in one padding element for every segment + if (SEGMENT_PADDING > 0) + { + offset += offset / SEGMENT_LENGTH; + } + + // Incorporating a block of padding partials every shared memory segment + return temp_storage + offset; + } + + + /** + * \brief Returns the location for the calling thread to begin sequential raking + */ + static __device__ __forceinline__ T* RakingPtr( + TempStorage &temp_storage, + int linear_tid) + { + return temp_storage + (linear_tid * (SEGMENT_LENGTH + SEGMENT_PADDING)); + } +}; + +} // CUB namespace +CUB_NS_POSTFIX // Optional outer namespace(s) + diff --git a/lib/kokkos/TPL/cub/block/block_reduce.cuh b/lib/kokkos/TPL/cub/block/block_reduce.cuh new file mode 100755 index 0000000000..ffdff73775 --- /dev/null +++ b/lib/kokkos/TPL/cub/block/block_reduce.cuh @@ -0,0 +1,563 @@ +/****************************************************************************** + * Copyright (c) 2011, Duane Merrill. All rights reserved. + * Copyright (c) 2011-2013, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the NVIDIA CORPORATION nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + ******************************************************************************/ + +/** + * \file + * The cub::BlockReduce class provides [collective](index.html#sec0) methods for computing a parallel reduction of items partitioned across a CUDA thread block. + */ + +#pragma once + +#include "specializations/block_reduce_raking.cuh" +#include "specializations/block_reduce_warp_reductions.cuh" +#include "../util_type.cuh" +#include "../thread/thread_operators.cuh" +#include "../util_namespace.cuh" + +/// Optional outer namespace(s) +CUB_NS_PREFIX + +/// CUB namespace +namespace cub { + + + +/****************************************************************************** + * Algorithmic variants + ******************************************************************************/ + +/** + * BlockReduceAlgorithm enumerates alternative algorithms for parallel + * reduction across a CUDA threadblock. + */ +enum BlockReduceAlgorithm +{ + + /** + * \par Overview + * An efficient "raking" reduction algorithm. Execution is comprised of + * three phases: + * -# Upsweep sequential reduction in registers (if threads contribute more + * than one input each). Each thread then places the partial reduction + * of its item(s) into shared memory. + * -# Upsweep sequential reduction in shared memory. Threads within a + * single warp rake across segments of shared partial reductions. + * -# A warp-synchronous Kogge-Stone style reduction within the raking warp. + * + * \par + * \image html block_reduce.png + *
\p BLOCK_REDUCE_RAKING data flow for a hypothetical 16-thread threadblock and 4-thread raking warp.
+ * + * \par Performance Considerations + * - Although this variant may suffer longer turnaround latencies when the + * GPU is under-occupied, it can often provide higher overall throughput + * across the GPU when suitably occupied. + */ + BLOCK_REDUCE_RAKING, + + + /** + * \par Overview + * A quick "tiled warp-reductions" reduction algorithm. Execution is + * comprised of four phases: + * -# Upsweep sequential reduction in registers (if threads contribute more + * than one input each). Each thread then places the partial reduction + * of its item(s) into shared memory. + * -# Compute a shallow, but inefficient warp-synchronous Kogge-Stone style + * reduction within each warp. + * -# A propagation phase where the warp reduction outputs in each warp are + * updated with the aggregate from each preceding warp. + * + * \par + * \image html block_scan_warpscans.png + *
\p BLOCK_REDUCE_WARP_REDUCTIONS data flow for a hypothetical 16-thread threadblock and 4-thread raking warp.
+ * + * \par Performance Considerations + * - Although this variant may suffer lower overall throughput across the + * GPU because due to a heavy reliance on inefficient warp-reductions, it + * can often provide lower turnaround latencies when the GPU is + * under-occupied. + */ + BLOCK_REDUCE_WARP_REDUCTIONS, +}; + + +/****************************************************************************** + * Block reduce + ******************************************************************************/ + +/** + * \brief The BlockReduce class provides [collective](index.html#sec0) methods for computing a parallel reduction of items partitioned across a CUDA thread block. ![](reduce_logo.png) + * \ingroup BlockModule + * + * \par Overview + * A reduction (or fold) + * uses a binary combining operator to compute a single aggregate from a list of input elements. + * + * \par + * Optionally, BlockReduce can be specialized by algorithm to accommodate different latency/throughput workload profiles: + * -# cub::BLOCK_REDUCE_RAKING. An efficient "raking" reduction algorithm. [More...](\ref cub::BlockReduceAlgorithm) + * -# cub::BLOCK_REDUCE_WARP_REDUCTIONS. A quick "tiled warp-reductions" reduction algorithm. [More...](\ref cub::BlockReduceAlgorithm) + * + * \tparam T Data type being reduced + * \tparam BLOCK_THREADS The thread block size in threads + * \tparam ALGORITHM [optional] cub::BlockReduceAlgorithm enumerator specifying the underlying algorithm to use (default: cub::BLOCK_REDUCE_RAKING) + * + * \par Performance Considerations + * - Very efficient (only one synchronization barrier). + * - Zero bank conflicts for most types. + * - Computation is slightly more efficient (i.e., having lower instruction overhead) for: + * - Summation (vs. generic reduction) + * - \p BLOCK_THREADS is a multiple of the architecture's warp size + * - Every thread has a valid input (i.e., full vs. partial-tiles) + * - See cub::BlockReduceAlgorithm for performance details regarding algorithmic alternatives + * + * \par A Simple Example + * \blockcollective{BlockReduce} + * \par + * The code snippet below illustrates a sum reduction of 512 integer items that + * are partitioned in a [blocked arrangement](index.html#sec5sec4) across 128 threads + * where each thread owns 4 consecutive items. + * \par + * \code + * #include + * + * __global__ void ExampleKernel(...) + * { + * // Specialize BlockReduce for 128 threads on type int + * typedef cub::BlockReduce BlockReduce; + * + * // Allocate shared memory for BlockReduce + * __shared__ typename BlockReduce::TempStorage temp_storage; + * + * // Obtain a segment of consecutive items that are blocked across threads + * int thread_data[4]; + * ... + * + * // Compute the block-wide sum for thread0 + * int aggregate = BlockReduce(temp_storage).Sum(thread_data); + * + * \endcode + * + */ +template < + typename T, + int BLOCK_THREADS, + BlockReduceAlgorithm ALGORITHM = BLOCK_REDUCE_RAKING> +class BlockReduce +{ +private: + + /****************************************************************************** + * Constants and typedefs + ******************************************************************************/ + + /// Internal specialization. + typedef typename If<(ALGORITHM == BLOCK_REDUCE_WARP_REDUCTIONS), + BlockReduceWarpReductions, + BlockReduceRaking >::Type InternalBlockReduce; + + /// Shared memory storage layout type for BlockReduce + typedef typename InternalBlockReduce::TempStorage _TempStorage; + + + /****************************************************************************** + * Utility methods + ******************************************************************************/ + + /// Internal storage allocator + __device__ __forceinline__ _TempStorage& PrivateStorage() + { + __shared__ _TempStorage private_storage; + return private_storage; + } + + + /****************************************************************************** + * Thread fields + ******************************************************************************/ + + /// Shared storage reference + _TempStorage &temp_storage; + + /// Linear thread-id + int linear_tid; + + +public: + + /// \smemstorage{BlockReduce} + struct TempStorage : Uninitialized<_TempStorage> {}; + + + /******************************************************************//** + * \name Collective constructors + *********************************************************************/ + //@{ + + /** + * \brief Collective constructor for 1D thread blocks using a private static allocation of shared memory as temporary storage. Threads are identified using threadIdx.x. + */ + __device__ __forceinline__ BlockReduce() + : + temp_storage(PrivateStorage()), + linear_tid(threadIdx.x) + {} + + + /** + * \brief Collective constructor for 1D thread blocks using the specified memory allocation as temporary storage. Threads are identified using threadIdx.x. + */ + __device__ __forceinline__ BlockReduce( + TempStorage &temp_storage) ///< [in] Reference to memory allocation having layout type TempStorage + : + temp_storage(temp_storage.Alias()), + linear_tid(threadIdx.x) + {} + + + /** + * \brief Collective constructor using a private static allocation of shared memory as temporary storage. Each thread is identified using the supplied linear thread identifier + */ + __device__ __forceinline__ BlockReduce( + int linear_tid) ///< [in] A suitable 1D thread-identifier for the calling thread (e.g., (threadIdx.y * blockDim.x) + linear_tid for 2D thread blocks) + : + temp_storage(PrivateStorage()), + linear_tid(linear_tid) + {} + + + /** + * \brief Collective constructor using the specified memory allocation as temporary storage. Each thread is identified using the supplied linear thread identifier. + */ + __device__ __forceinline__ BlockReduce( + TempStorage &temp_storage, ///< [in] Reference to memory allocation having layout type TempStorage + int linear_tid) ///< [in] [optional] A suitable 1D thread-identifier for the calling thread (e.g., (threadIdx.y * blockDim.x) + linear_tid for 2D thread blocks) + : + temp_storage(temp_storage.Alias()), + linear_tid(linear_tid) + {} + + + + //@} end member group + /******************************************************************//** + * \name Generic reductions + *********************************************************************/ + //@{ + + + /** + * \brief Computes a block-wide reduction for thread0 using the specified binary reduction functor. Each thread contributes one input element. + * + * The return value is undefined in threads other than thread0. + * + * Supports non-commutative reduction operators. + * + * \smemreuse + * + * The code snippet below illustrates a max reduction of 128 integer items that + * are partitioned across 128 threads. + * \par + * \code + * #include + * + * __global__ void ExampleKernel(...) + * { + * // Specialize BlockReduce for 128 threads on type int + * typedef cub::BlockReduce BlockReduce; + * + * // Allocate shared memory for BlockReduce + * __shared__ typename BlockReduce::TempStorage temp_storage; + * + * // Each thread obtains an input item + * int thread_data; + * ... + * + * // Compute the block-wide max for thread0 + * int aggregate = BlockReduce(temp_storage).Reduce(thread_data, cub::Max()); + * + * \endcode + * + * \tparam ReductionOp [inferred] Binary reduction operator type having member T operator()(const T &a, const T &b) + */ + template + __device__ __forceinline__ T Reduce( + T input, ///< [in] Calling thread's input + ReductionOp reduction_op) ///< [in] Binary reduction operator + { + return InternalBlockReduce(temp_storage, linear_tid).template Reduce(input, BLOCK_THREADS, reduction_op); + } + + + /** + * \brief Computes a block-wide reduction for thread0 using the specified binary reduction functor. Each thread contributes an array of consecutive input elements. + * + * The return value is undefined in threads other than thread0. + * + * Supports non-commutative reduction operators. + * + * \blocked + * + * \smemreuse + * + * The code snippet below illustrates a max reduction of 512 integer items that + * are partitioned in a [blocked arrangement](index.html#sec5sec4) across 128 threads + * where each thread owns 4 consecutive items. + * \par + * \code + * #include + * + * __global__ void ExampleKernel(...) + * { + * // Specialize BlockReduce for 128 threads on type int + * typedef cub::BlockReduce BlockReduce; + * + * // Allocate shared memory for BlockReduce + * __shared__ typename BlockReduce::TempStorage temp_storage; + * + * // Obtain a segment of consecutive items that are blocked across threads + * int thread_data[4]; + * ... + * + * // Compute the block-wide max for thread0 + * int aggregate = BlockReduce(temp_storage).Reduce(thread_data, cub::Max()); + * + * \endcode + * + * \tparam ITEMS_PER_THREAD [inferred] The number of consecutive items partitioned onto each thread. + * \tparam ReductionOp [inferred] Binary reduction operator type having member T operator()(const T &a, const T &b) + */ + template < + int ITEMS_PER_THREAD, + typename ReductionOp> + __device__ __forceinline__ T Reduce( + T (&inputs)[ITEMS_PER_THREAD], ///< [in] Calling thread's input segment + ReductionOp reduction_op) ///< [in] Binary reduction operator + { + // Reduce partials + T partial = ThreadReduce(inputs, reduction_op); + return Reduce(partial, reduction_op); + } + + + /** + * \brief Computes a block-wide reduction for thread0 using the specified binary reduction functor. The first \p num_valid threads each contribute one input element. + * + * The return value is undefined in threads other than thread0. + * + * Supports non-commutative reduction operators. + * + * \blocked + * + * \smemreuse + * + * The code snippet below illustrates a max reduction of a partially-full tile of integer items that + * are partitioned across 128 threads. + * \par + * \code + * #include + * + * __global__ void ExampleKernel(int num_valid, ...) + * { + * // Specialize BlockReduce for 128 threads on type int + * typedef cub::BlockReduce BlockReduce; + * + * // Allocate shared memory for BlockReduce + * __shared__ typename BlockReduce::TempStorage temp_storage; + * + * // Each thread obtains an input item + * int thread_data; + * if (threadIdx.x < num_valid) thread_data = ... + * + * // Compute the block-wide max for thread0 + * int aggregate = BlockReduce(temp_storage).Reduce(thread_data, cub::Max(), num_valid); + * + * \endcode + * + * \tparam ReductionOp [inferred] Binary reduction operator type having member T operator()(const T &a, const T &b) + */ + template + __device__ __forceinline__ T Reduce( + T input, ///< [in] Calling thread's input + ReductionOp reduction_op, ///< [in] Binary reduction operator + int num_valid) ///< [in] Number of threads containing valid elements (may be less than BLOCK_THREADS) + { + // Determine if we scan skip bounds checking + if (num_valid >= BLOCK_THREADS) + { + return InternalBlockReduce(temp_storage, linear_tid).template Reduce(input, num_valid, reduction_op); + } + else + { + return InternalBlockReduce(temp_storage, linear_tid).template Reduce(input, num_valid, reduction_op); + } + } + + + //@} end member group + /******************************************************************//** + * \name Summation reductions + *********************************************************************/ + //@{ + + + /** + * \brief Computes a block-wide reduction for thread0 using addition (+) as the reduction operator. Each thread contributes one input element. + * + * The return value is undefined in threads other than thread0. + * + * \smemreuse + * + * The code snippet below illustrates a sum reduction of 128 integer items that + * are partitioned across 128 threads. + * \par + * \code + * #include + * + * __global__ void ExampleKernel(...) + * { + * // Specialize BlockReduce for 128 threads on type int + * typedef cub::BlockReduce BlockReduce; + * + * // Allocate shared memory for BlockReduce + * __shared__ typename BlockReduce::TempStorage temp_storage; + * + * // Each thread obtains an input item + * int thread_data; + * ... + * + * // Compute the block-wide sum for thread0 + * int aggregate = BlockReduce(temp_storage).Sum(thread_data); + * + * \endcode + * + */ + __device__ __forceinline__ T Sum( + T input) ///< [in] Calling thread's input + { + return InternalBlockReduce(temp_storage, linear_tid).template Sum(input, BLOCK_THREADS); + } + + /** + * \brief Computes a block-wide reduction for thread0 using addition (+) as the reduction operator. Each thread contributes an array of consecutive input elements. + * + * The return value is undefined in threads other than thread0. + * + * \smemreuse + * + * The code snippet below illustrates a sum reduction of 512 integer items that + * are partitioned in a [blocked arrangement](index.html#sec5sec4) across 128 threads + * where each thread owns 4 consecutive items. + * \par + * \code + * #include + * + * __global__ void ExampleKernel(...) + * { + * // Specialize BlockReduce for 128 threads on type int + * typedef cub::BlockReduce BlockReduce; + * + * // Allocate shared memory for BlockReduce + * __shared__ typename BlockReduce::TempStorage temp_storage; + * + * // Obtain a segment of consecutive items that are blocked across threads + * int thread_data[4]; + * ... + * + * // Compute the block-wide sum for thread0 + * int aggregate = BlockReduce(temp_storage).Sum(thread_data); + * + * \endcode + * + * \tparam ITEMS_PER_THREAD [inferred] The number of consecutive items partitioned onto each thread. + */ + template + __device__ __forceinline__ T Sum( + T (&inputs)[ITEMS_PER_THREAD]) ///< [in] Calling thread's input segment + { + // Reduce partials + T partial = ThreadReduce(inputs, cub::Sum()); + return Sum(partial); + } + + + /** + * \brief Computes a block-wide reduction for thread0 using addition (+) as the reduction operator. The first \p num_valid threads each contribute one input element. + * + * The return value is undefined in threads other than thread0. + * + * \smemreuse + * + * The code snippet below illustrates a sum reduction of a partially-full tile of integer items that + * are partitioned across 128 threads. + * \par + * \code + * #include + * + * __global__ void ExampleKernel(int num_valid, ...) + * { + * // Specialize BlockReduce for 128 threads on type int + * typedef cub::BlockReduce BlockReduce; + * + * // Allocate shared memory for BlockReduce + * __shared__ typename BlockReduce::TempStorage temp_storage; + * + * // Each thread obtains an input item (up to num_items) + * int thread_data; + * if (threadIdx.x < num_valid) + * thread_data = ... + * + * // Compute the block-wide sum for thread0 + * int aggregate = BlockReduce(temp_storage).Sum(thread_data, num_valid); + * + * \endcode + * + */ + __device__ __forceinline__ T Sum( + T input, ///< [in] Calling thread's input + int num_valid) ///< [in] Number of threads containing valid elements (may be less than BLOCK_THREADS) + { + // Determine if we scan skip bounds checking + if (num_valid >= BLOCK_THREADS) + { + return InternalBlockReduce(temp_storage, linear_tid).template Sum(input, num_valid); + } + else + { + return InternalBlockReduce(temp_storage, linear_tid).template Sum(input, num_valid); + } + } + + + //@} end member group +}; + +} // CUB namespace +CUB_NS_POSTFIX // Optional outer namespace(s) + diff --git a/lib/kokkos/TPL/cub/block/block_scan.cuh b/lib/kokkos/TPL/cub/block/block_scan.cuh new file mode 100755 index 0000000000..1c1a2dac81 --- /dev/null +++ b/lib/kokkos/TPL/cub/block/block_scan.cuh @@ -0,0 +1,2233 @@ +/****************************************************************************** + * Copyright (c) 2011, Duane Merrill. All rights reserved. + * Copyright (c) 2011-2013, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the NVIDIA CORPORATION nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + ******************************************************************************/ + +/** + * \file + * The cub::BlockScan class provides [collective](index.html#sec0) methods for computing a parallel prefix sum/scan of items partitioned across a CUDA thread block. + */ + +#pragma once + +#include "specializations/block_scan_raking.cuh" +#include "specializations/block_scan_warp_scans.cuh" +#include "../util_arch.cuh" +#include "../util_type.cuh" +#include "../util_namespace.cuh" + +/// Optional outer namespace(s) +CUB_NS_PREFIX + +/// CUB namespace +namespace cub { + + +/****************************************************************************** + * Algorithmic variants + ******************************************************************************/ + +/** + * \brief BlockScanAlgorithm enumerates alternative algorithms for cub::BlockScan to compute a parallel prefix scan across a CUDA thread block. + */ +enum BlockScanAlgorithm +{ + + /** + * \par Overview + * An efficient "raking reduce-then-scan" prefix scan algorithm. Execution is comprised of five phases: + * -# Upsweep sequential reduction in registers (if threads contribute more than one input each). Each thread then places the partial reduction of its item(s) into shared memory. + * -# Upsweep sequential reduction in shared memory. Threads within a single warp rake across segments of shared partial reductions. + * -# A warp-synchronous Kogge-Stone style exclusive scan within the raking warp. + * -# Downsweep sequential exclusive scan in shared memory. Threads within a single warp rake across segments of shared partial reductions, seeded with the warp-scan output. + * -# Downsweep sequential scan in registers (if threads contribute more than one input), seeded with the raking scan output. + * + * \par + * \image html block_scan_raking.png + *
\p BLOCK_SCAN_RAKING data flow for a hypothetical 16-thread threadblock and 4-thread raking warp.
+ * + * \par Performance Considerations + * - Although this variant may suffer longer turnaround latencies when the + * GPU is under-occupied, it can often provide higher overall throughput + * across the GPU when suitably occupied. + */ + BLOCK_SCAN_RAKING, + + + /** + * \par Overview + * Similar to cub::BLOCK_SCAN_RAKING, but with fewer shared memory reads at + * the expense of higher register pressure. Raking threads preserve their + * "upsweep" segment of values in registers while performing warp-synchronous + * scan, allowing the "downsweep" not to re-read them from shared memory. + */ + BLOCK_SCAN_RAKING_MEMOIZE, + + + /** + * \par Overview + * A quick "tiled warpscans" prefix scan algorithm. Execution is comprised of four phases: + * -# Upsweep sequential reduction in registers (if threads contribute more than one input each). Each thread then places the partial reduction of its item(s) into shared memory. + * -# Compute a shallow, but inefficient warp-synchronous Kogge-Stone style scan within each warp. + * -# A propagation phase where the warp scan outputs in each warp are updated with the aggregate from each preceding warp. + * -# Downsweep sequential scan in registers (if threads contribute more than one input), seeded with the raking scan output. + * + * \par + * \image html block_scan_warpscans.png + *
\p BLOCK_SCAN_WARP_SCANS data flow for a hypothetical 16-thread threadblock and 4-thread raking warp.
+ * + * \par Performance Considerations + * - Although this variant may suffer lower overall throughput across the + * GPU because due to a heavy reliance on inefficient warpscans, it can + * often provide lower turnaround latencies when the GPU is under-occupied. + */ + BLOCK_SCAN_WARP_SCANS, +}; + + +/****************************************************************************** + * Block scan + ******************************************************************************/ + +/** + * \brief The BlockScan class provides [collective](index.html#sec0) methods for computing a parallel prefix sum/scan of items partitioned across a CUDA thread block. ![](block_scan_logo.png) + * \ingroup BlockModule + * + * \par Overview + * Given a list of input elements and a binary reduction operator, a [prefix scan](http://en.wikipedia.org/wiki/Prefix_sum) + * produces an output list where each element is computed to be the reduction + * of the elements occurring earlier in the input list. Prefix sum + * connotes a prefix scan with the addition operator. The term \em inclusive indicates + * that the ith output reduction incorporates the ith input. + * The term \em exclusive indicates the ith input is not incorporated into + * the ith output reduction. + * + * \par + * Optionally, BlockScan can be specialized by algorithm to accommodate different latency/throughput workload profiles: + * -# cub::BLOCK_SCAN_RAKING. An efficient "raking reduce-then-scan" prefix scan algorithm. [More...](\ref cub::BlockScanAlgorithm) + * -# cub::BLOCK_SCAN_WARP_SCANS. A quick "tiled warpscans" prefix scan algorithm. [More...](\ref cub::BlockScanAlgorithm) + * + * \tparam T Data type being scanned + * \tparam BLOCK_THREADS The thread block size in threads + * \tparam ALGORITHM [optional] cub::BlockScanAlgorithm enumerator specifying the underlying algorithm to use (default: cub::BLOCK_SCAN_RAKING) + * + * \par A Simple Example + * \blockcollective{BlockScan} + * \par + * The code snippet below illustrates an exclusive prefix sum of 512 integer items that + * are partitioned in a [blocked arrangement](index.html#sec5sec4) across 128 threads + * where each thread owns 4 consecutive items. + * \par + * \code + * #include + * + * __global__ void ExampleKernel(...) + * { + * // Specialize BlockScan for 128 threads on type int + * typedef cub::BlockScan BlockScan; + * + * // Allocate shared memory for BlockScan + * __shared__ typename BlockScan::TempStorage temp_storage; + * + * // Obtain a segment of consecutive items that are blocked across threads + * int thread_data[4]; + * ... + * + * // Collectively compute the block-wide exclusive prefix sum + * BlockScan(temp_storage).ExclusiveSum(thread_data, thread_data); + * + * \endcode + * \par + * Suppose the set of input \p thread_data across the block of threads is + * { [1,1,1,1], [1,1,1,1], ..., [1,1,1,1] }. + * The corresponding output \p thread_data in those threads will be + * { [0,1,2,3], [4,5,6,7], ..., [508,509,510,511] }. + * + * \par Performance Considerations + * - Uses special instructions when applicable (e.g., warp \p SHFL) + * - Uses synchronization-free communication between warp lanes when applicable + * - Uses only one or two block-wide synchronization barriers (depending on + * algorithm selection) + * - Zero bank conflicts for most types + * - Computation is slightly more efficient (i.e., having lower instruction overhead) for: + * - Prefix sum variants (vs. generic scan) + * - Exclusive variants (vs. inclusive) + * - \p BLOCK_THREADS is a multiple of the architecture's warp size + * - See cub::BlockScanAlgorithm for performance details regarding algorithmic alternatives + * + */ +template < + typename T, + int BLOCK_THREADS, + BlockScanAlgorithm ALGORITHM = BLOCK_SCAN_RAKING> +class BlockScan +{ +private: + + /****************************************************************************** + * Constants and typedefs + ******************************************************************************/ + + /** + * Ensure the template parameterization meets the requirements of the + * specified algorithm. Currently, the BLOCK_SCAN_WARP_SCANS policy + * cannot be used with threadblock sizes not a multiple of the + * architectural warp size. + */ + static const BlockScanAlgorithm SAFE_ALGORITHM = + ((ALGORITHM == BLOCK_SCAN_WARP_SCANS) && (BLOCK_THREADS % PtxArchProps::WARP_THREADS != 0)) ? + BLOCK_SCAN_RAKING : + ALGORITHM; + + /// Internal specialization. + typedef typename If<(SAFE_ALGORITHM == BLOCK_SCAN_WARP_SCANS), + BlockScanWarpScans, + BlockScanRaking >::Type InternalBlockScan; + + + /// Shared memory storage layout type for BlockScan + typedef typename InternalBlockScan::TempStorage _TempStorage; + + + /****************************************************************************** + * Thread fields + ******************************************************************************/ + + /// Shared storage reference + _TempStorage &temp_storage; + + /// Linear thread-id + int linear_tid; + + + /****************************************************************************** + * Utility methods + ******************************************************************************/ + + /// Internal storage allocator + __device__ __forceinline__ _TempStorage& PrivateStorage() + { + __shared__ _TempStorage private_storage; + return private_storage; + } + + +public: + + /// \smemstorage{BlockScan} + struct TempStorage : Uninitialized<_TempStorage> {}; + + + /******************************************************************//** + * \name Collective constructors + *********************************************************************/ + //@{ + + /** + * \brief Collective constructor for 1D thread blocks using a private static allocation of shared memory as temporary storage. Threads are identified using threadIdx.x. + */ + __device__ __forceinline__ BlockScan() + : + temp_storage(PrivateStorage()), + linear_tid(threadIdx.x) + {} + + + /** + * \brief Collective constructor for 1D thread blocks using the specified memory allocation as temporary storage. Threads are identified using threadIdx.x. + */ + __device__ __forceinline__ BlockScan( + TempStorage &temp_storage) ///< [in] Reference to memory allocation having layout type TempStorage + : + temp_storage(temp_storage.Alias()), + linear_tid(threadIdx.x) + {} + + + /** + * \brief Collective constructor using a private static allocation of shared memory as temporary storage. Each thread is identified using the supplied linear thread identifier + */ + __device__ __forceinline__ BlockScan( + int linear_tid) ///< [in] A suitable 1D thread-identifier for the calling thread (e.g., (threadIdx.y * blockDim.x) + linear_tid for 2D thread blocks) + : + temp_storage(PrivateStorage()), + linear_tid(linear_tid) + {} + + + /** + * \brief Collective constructor using the specified memory allocation as temporary storage. Each thread is identified using the supplied linear thread identifier. + */ + __device__ __forceinline__ BlockScan( + TempStorage &temp_storage, ///< [in] Reference to memory allocation having layout type TempStorage + int linear_tid) ///< [in] [optional] A suitable 1D thread-identifier for the calling thread (e.g., (threadIdx.y * blockDim.x) + linear_tid for 2D thread blocks) + : + temp_storage(temp_storage.Alias()), + linear_tid(linear_tid) + {} + + + + //@} end member group + /******************************************************************//** + * \name Exclusive prefix sum operations + *********************************************************************/ + //@{ + + + /** + * \brief Computes an exclusive block-wide prefix scan using addition (+) as the scan operator. Each thread contributes one input element. + * + * \blocked + * + * \smemreuse + * + * The code snippet below illustrates an exclusive prefix sum of 128 integer items that + * are partitioned across 128 threads. + * \par + * \code + * #include + * + * __global__ void ExampleKernel(...) + * { + * // Specialize BlockScan for 128 threads on type int + * typedef cub::BlockScan BlockScan; + * + * // Allocate shared memory for BlockScan + * __shared__ typename BlockScan::TempStorage temp_storage; + * + * // Obtain input item for each thread + * int thread_data; + * ... + * + * // Collectively compute the block-wide exclusive prefix sum + * BlockScan(temp_storage).ExclusiveSum(thread_data, thread_data); + * + * \endcode + * \par + * Suppose the set of input \p thread_data across the block of threads is 1, 1, ..., 1. The + * corresponding output \p thread_data in those threads will be 0, 1, ..., 127. + * + */ + __device__ __forceinline__ void ExclusiveSum( + T input, ///< [in] Calling thread's input item + T &output) ///< [out] Calling thread's output item (may be aliased to \p input) + { + T block_aggregate; + InternalBlockScan(temp_storage, linear_tid).ExclusiveSum(input, output, block_aggregate); + } + + + /** + * \brief Computes an exclusive block-wide prefix scan using addition (+) as the scan operator. Each thread contributes one input element. Also provides every thread with the block-wide \p block_aggregate of all inputs. + * + * \blocked + * + * \smemreuse + * + * The code snippet below illustrates an exclusive prefix sum of 128 integer items that + * are partitioned across 128 threads. + * \par + * \code + * #include + * + * __global__ void ExampleKernel(...) + * { + * // Specialize BlockScan for 128 threads on type int + * typedef cub::BlockScan BlockScan; + * + * // Allocate shared memory for BlockScan + * __shared__ typename BlockScan::TempStorage temp_storage; + * + * // Obtain input item for each thread + * int thread_data; + * ... + * + * // Collectively compute the block-wide exclusive prefix sum + * int block_aggregate; + * BlockScan(temp_storage).ExclusiveSum(thread_data, thread_data, block_aggregate); + * + * \endcode + * \par + * Suppose the set of input \p thread_data across the block of threads is 1, 1, ..., 1. The + * corresponding output \p thread_data in those threads will be 0, 1, ..., 127. + * Furthermore the value \p 128 will be stored in \p block_aggregate for all threads. + * + */ + __device__ __forceinline__ void ExclusiveSum( + T input, ///< [in] Calling thread's input item + T &output, ///< [out] Calling thread's output item (may be aliased to \p input) + T &block_aggregate) ///< [out] block-wide aggregate reduction of input items + { + InternalBlockScan(temp_storage, linear_tid).ExclusiveSum(input, output, block_aggregate); + } + + + /** + * \brief Computes an exclusive block-wide prefix scan using addition (+) as the scan operator. Each thread contributes one input element. Instead of using 0 as the block-wide prefix, the call-back functor \p block_prefix_op is invoked by the first warp in the block, and the value returned by lane0 in that warp is used as the "seed" value that logically prefixes the threadblock's scan inputs. Also provides every thread with the block-wide \p block_aggregate of all inputs. + * + * The \p block_prefix_op functor must implement a member function T operator()(T block_aggregate). + * The functor's input parameter \p block_aggregate is the same value also returned by the scan operation. + * The functor will be invoked by the first warp of threads in the block, however only the return value from + * lane0 is applied as the block-wide prefix. Can be stateful. + * + * \blocked + * + * \smemreuse + * + * The code snippet below illustrates a single thread block that progressively + * computes an exclusive prefix sum over multiple "tiles" of input using a + * prefix functor to maintain a running total between block-wide scans. Each tile consists + * of 128 integer items that are partitioned across 128 threads. + * \par + * \code + * #include + * + * // A stateful callback functor that maintains a running prefix to be applied + * // during consecutive scan operations. + * struct BlockPrefixOp + * { + * // Running prefix + * int running_total; + * + * // Constructor + * __device__ BlockPrefixOp(int running_total) : running_total(running_total) {} + * + * // Callback operator to be entered by the first warp of threads in the block. + * // Thread-0 is responsible for returning a value for seeding the block-wide scan. + * __device__ int operator()(int block_aggregate) + * { + * int old_prefix = running_total; + * running_total += block_aggregate; + * return old_prefix; + * } + * }; + * + * __global__ void ExampleKernel(int *d_data, int num_items, ...) + * { + * // Specialize BlockScan for 128 threads + * typedef cub::BlockScan BlockScan; + * + * // Allocate shared memory for BlockScan + * __shared__ typename BlockScan::TempStorage temp_storage; + * + * // Initialize running total + * BlockPrefixOp prefix_op(0); + * + * // Have the block iterate over segments of items + * for (int block_offset = 0; block_offset < num_items; block_offset += 128) + * { + * // Load a segment of consecutive items that are blocked across threads + * int thread_data = d_data[block_offset]; + * + * // Collectively compute the block-wide exclusive prefix sum + * int block_aggregate; + * BlockScan(temp_storage).ExclusiveSum( + * thread_data, thread_data, block_aggregate, prefix_op); + * __syncthreads(); + * + * // Store scanned items to output segment + * d_data[block_offset] = thread_data; + * } + * \endcode + * \par + * Suppose the input \p d_data is 1, 1, 1, 1, 1, 1, 1, 1, .... + * The corresponding output for the first segment will be 0, 1, ..., 127. + * The output for the second segment will be 128, 129, ..., 255. Furthermore, + * the value \p 128 will be stored in \p block_aggregate for all threads after each scan. + * + * \tparam BlockPrefixOp [inferred] Call-back functor type having member T operator()(T block_aggregate) + */ + template + __device__ __forceinline__ void ExclusiveSum( + T input, ///< [in] Calling thread's input item + T &output, ///< [out] Calling thread's output item (may be aliased to \p input) + T &block_aggregate, ///< [out] block-wide aggregate reduction of input items (exclusive of the \p block_prefix_op value) + BlockPrefixOp &block_prefix_op) ///< [in-out] [warp0 only] Call-back functor for specifying a block-wide prefix to be applied to all inputs. + { + InternalBlockScan(temp_storage, linear_tid).ExclusiveSum(input, output, block_aggregate, block_prefix_op); + } + + + //@} end member group + /******************************************************************//** + * \name Exclusive prefix sum operations (multiple data per thread) + *********************************************************************/ + //@{ + + + /** + * \brief Computes an exclusive block-wide prefix scan using addition (+) as the scan operator. Each thread contributes an array of consecutive input elements. + * + * \blocked + * + * \smemreuse + * + * The code snippet below illustrates an exclusive prefix sum of 512 integer items that + * are partitioned in a [blocked arrangement](index.html#sec5sec4) across 128 threads + * where each thread owns 4 consecutive items. + * \par + * \code + * #include + * + * __global__ void ExampleKernel(...) + * { + * // Specialize BlockScan for 128 threads on type int + * typedef cub::BlockScan BlockScan; + * + * // Allocate shared memory for BlockScan + * __shared__ typename BlockScan::TempStorage temp_storage; + * + * // Obtain a segment of consecutive items that are blocked across threads + * int thread_data[4]; + * ... + * + * // Collectively compute the block-wide exclusive prefix sum + * BlockScan(temp_storage).ExclusiveSum(thread_data, thread_data); + * + * \endcode + * \par + * Suppose the set of input \p thread_data across the block of threads is { [1,1,1,1], [1,1,1,1], ..., [1,1,1,1] }. The + * corresponding output \p thread_data in those threads will be { [0,1,2,3], [4,5,6,7], ..., [508,509,510,511] }. + * + * \tparam ITEMS_PER_THREAD [inferred] The number of consecutive items partitioned onto each thread. + */ + template + __device__ __forceinline__ void ExclusiveSum( + T (&input)[ITEMS_PER_THREAD], ///< [in] Calling thread's input items + T (&output)[ITEMS_PER_THREAD]) ///< [out] Calling thread's output items (may be aliased to \p input) + { + // Reduce consecutive thread items in registers + Sum scan_op; + T thread_partial = ThreadReduce(input, scan_op); + + // Exclusive threadblock-scan + ExclusiveSum(thread_partial, thread_partial); + + // Exclusive scan in registers with prefix + ThreadScanExclusive(input, output, scan_op, thread_partial); + } + + + /** + * \brief Computes an exclusive block-wide prefix scan using addition (+) as the scan operator. Each thread contributes an array of consecutive input elements. Also provides every thread with the block-wide \p block_aggregate of all inputs. + * + * \blocked + * + * \smemreuse + * + * The code snippet below illustrates an exclusive prefix sum of 512 integer items that + * are partitioned in a [blocked arrangement](index.html#sec5sec4) across 128 threads + * where each thread owns 4 consecutive items. + * \par + * \code + * #include + * + * __global__ void ExampleKernel(...) + * { + * // Specialize BlockScan for 128 threads on type int + * typedef cub::BlockScan BlockScan; + * + * // Allocate shared memory for BlockScan + * __shared__ typename BlockScan::TempStorage temp_storage; + * + * // Obtain a segment of consecutive items that are blocked across threads + * int thread_data[4]; + * ... + * + * // Collectively compute the block-wide exclusive prefix sum + * int block_aggregate; + * BlockScan(temp_storage).ExclusiveSum(thread_data, thread_data, block_aggregate); + * + * \endcode + * \par + * Suppose the set of input \p thread_data across the block of threads is { [1,1,1,1], [1,1,1,1], ..., [1,1,1,1] }. The + * corresponding output \p thread_data in those threads will be { [0,1,2,3], [4,5,6,7], ..., [508,509,510,511] }. + * Furthermore the value \p 512 will be stored in \p block_aggregate for all threads. + * + * \tparam ITEMS_PER_THREAD [inferred] The number of consecutive items partitioned onto each thread. + */ + template + __device__ __forceinline__ void ExclusiveSum( + T (&input)[ITEMS_PER_THREAD], ///< [in] Calling thread's input items + T (&output)[ITEMS_PER_THREAD], ///< [out] Calling thread's output items (may be aliased to \p input) + T &block_aggregate) ///< [out] block-wide aggregate reduction of input items + { + // Reduce consecutive thread items in registers + Sum scan_op; + T thread_partial = ThreadReduce(input, scan_op); + + // Exclusive threadblock-scan + ExclusiveSum(thread_partial, thread_partial, block_aggregate); + + // Exclusive scan in registers with prefix + ThreadScanExclusive(input, output, scan_op, thread_partial); + } + + + /** + * \brief Computes an exclusive block-wide prefix scan using addition (+) as the scan operator. Each thread contributes an array of consecutive input elements. Instead of using 0 as the block-wide prefix, the call-back functor \p block_prefix_op is invoked by the first warp in the block, and the value returned by lane0 in that warp is used as the "seed" value that logically prefixes the threadblock's scan inputs. Also provides every thread with the block-wide \p block_aggregate of all inputs. + * + * The \p block_prefix_op functor must implement a member function T operator()(T block_aggregate). + * The functor's input parameter \p block_aggregate is the same value also returned by the scan operation. + * The functor will be invoked by the first warp of threads in the block, however only the return value from + * lane0 is applied as the block-wide prefix. Can be stateful. + * + * \blocked + * + * \smemreuse + * + * The code snippet below illustrates a single thread block that progressively + * computes an exclusive prefix sum over multiple "tiles" of input using a + * prefix functor to maintain a running total between block-wide scans. Each tile consists + * of 512 integer items that are partitioned in a [blocked arrangement](index.html#sec5sec4) + * across 128 threads where each thread owns 4 consecutive items. + * \par + * \code + * #include + * + * // A stateful callback functor that maintains a running prefix to be applied + * // during consecutive scan operations. + * struct BlockPrefixOp + * { + * // Running prefix + * int running_total; + * + * // Constructor + * __device__ BlockPrefixOp(int running_total) : running_total(running_total) {} + * + * // Callback operator to be entered by the first warp of threads in the block. + * // Thread-0 is responsible for returning a value for seeding the block-wide scan. + * __device__ int operator()(int block_aggregate) + * { + * int old_prefix = running_total; + * running_total += block_aggregate; + * return old_prefix; + * } + * }; + * + * __global__ void ExampleKernel(int *d_data, int num_items, ...) + * { + * // Specialize BlockLoad, BlockStore, and BlockScan for 128 threads, 4 ints per thread + * typedef cub::BlockLoad BlockLoad; + * typedef cub::BlockStore BlockStore; + * typedef cub::BlockScan BlockScan; + * + * // Allocate aliased shared memory for BlockLoad, BlockStore, and BlockScan + * __shared__ union { + * typename BlockLoad::TempStorage load; + * typename BlockScan::TempStorage scan; + * typename BlockStore::TempStorage store; + * } temp_storage; + * + * // Initialize running total + * BlockPrefixOp prefix_op(0); + * + * // Have the block iterate over segments of items + * for (int block_offset = 0; block_offset < num_items; block_offset += 128 * 4) + * { + * // Load a segment of consecutive items that are blocked across threads + * int thread_data[4]; + * BlockLoad(temp_storage.load).Load(d_data + block_offset, thread_data); + * __syncthreads(); + * + * // Collectively compute the block-wide exclusive prefix sum + * int block_aggregate; + * BlockScan(temp_storage.scan).ExclusiveSum( + * thread_data, thread_data, block_aggregate, prefix_op); + * __syncthreads(); + * + * // Store scanned items to output segment + * BlockStore(temp_storage.store).Store(d_data + block_offset, thread_data); + * __syncthreads(); + * } + * \endcode + * \par + * Suppose the input \p d_data is 1, 1, 1, 1, 1, 1, 1, 1, .... + * The corresponding output for the first segment will be 0, 1, 2, 3, ..., 510, 511. + * The output for the second segment will be 512, 513, 514, 515, ..., 1022, 1023. Furthermore, + * the value \p 512 will be stored in \p block_aggregate for all threads after each scan. + * + * \tparam ITEMS_PER_THREAD [inferred] The number of consecutive items partitioned onto each thread. + * \tparam BlockPrefixOp [inferred] Call-back functor type having member T operator()(T block_aggregate) + */ + template < + int ITEMS_PER_THREAD, + typename BlockPrefixOp> + __device__ __forceinline__ void ExclusiveSum( + T (&input)[ITEMS_PER_THREAD], ///< [in] Calling thread's input items + T (&output)[ITEMS_PER_THREAD], ///< [out] Calling thread's output items (may be aliased to \p input) + T &block_aggregate, ///< [out] block-wide aggregate reduction of input items (exclusive of the \p block_prefix_op value) + BlockPrefixOp &block_prefix_op) ///< [in-out] [warp0 only] Call-back functor for specifying a block-wide prefix to be applied to all inputs. + { + // Reduce consecutive thread items in registers + Sum scan_op; + T thread_partial = ThreadReduce(input, scan_op); + + // Exclusive threadblock-scan + ExclusiveSum(thread_partial, thread_partial, block_aggregate, block_prefix_op); + + // Exclusive scan in registers with prefix + ThreadScanExclusive(input, output, scan_op, thread_partial); + } + + + + //@} end member group // Inclusive prefix sums + /******************************************************************//** + * \name Exclusive prefix scan operations + *********************************************************************/ + //@{ + + + /** + * \brief Computes an exclusive block-wide prefix scan using the specified binary \p scan_op functor. Each thread contributes one input element. + * + * Supports non-commutative scan operators. + * + * \blocked + * + * \smemreuse + * + * The code snippet below illustrates an exclusive prefix max scan of 128 integer items that + * are partitioned across 128 threads. + * \par + * \code + * #include + * + * __global__ void ExampleKernel(...) + * { + * // Specialize BlockScan for 128 threads on type int + * typedef cub::BlockScan BlockScan; + * + * // Allocate shared memory for BlockScan + * __shared__ typename BlockScan::TempStorage temp_storage; + * + * // Obtain input item for each thread + * int thread_data; + * ... + * + * // Collectively compute the block-wide exclusive prefix max scan + * BlockScan(temp_storage).ExclusiveScan(thread_data, thread_data, INT_MIN, cub::Max()); + * + * \endcode + * \par + * Suppose the set of input \p thread_data across the block of threads is 0, -1, 2, -3, ..., 126, -127. The + * corresponding output \p thread_data in those threads will be INT_MIN, 0, 0, 2, ..., 124, 126. + * + * \tparam ScanOp [inferred] Binary scan operator type having member T operator()(const T &a, const T &b) + */ + template + __device__ __forceinline__ void ExclusiveScan( + T input, ///< [in] Calling thread's input item + T &output, ///< [out] Calling thread's output item (may be aliased to \p input) + T identity, ///< [in] Identity value + ScanOp scan_op) ///< [in] Binary scan operator + { + T block_aggregate; + InternalBlockScan(temp_storage, linear_tid).ExclusiveScan(input, output, identity, scan_op, block_aggregate); + } + + + /** + * \brief Computes an exclusive block-wide prefix scan using the specified binary \p scan_op functor. Each thread contributes one input element. Also provides every thread with the block-wide \p block_aggregate of all inputs. + * + * Supports non-commutative scan operators. + * + * \blocked + * + * \smemreuse + * + * The code snippet below illustrates an exclusive prefix max scan of 128 integer items that + * are partitioned across 128 threads. + * \par + * \code + * #include + * + * __global__ void ExampleKernel(...) + * { + * // Specialize BlockScan for 128 threads on type int + * typedef cub::BlockScan BlockScan; + * + * // Allocate shared memory for BlockScan + * __shared__ typename BlockScan::TempStorage temp_storage; + * + * // Obtain input item for each thread + * int thread_data; + * ... + * + * // Collectively compute the block-wide exclusive prefix max scan + * int block_aggregate; + * BlockScan(temp_storage).ExclusiveScan(thread_data, thread_data, INT_MIN, cub::Max(), block_aggregate); + * + * \endcode + * \par + * Suppose the set of input \p thread_data across the block of threads is 0, -1, 2, -3, ..., 126, -127. The + * corresponding output \p thread_data in those threads will be INT_MIN, 0, 0, 2, ..., 124, 126. + * Furthermore the value \p 126 will be stored in \p block_aggregate for all threads. + * + * \tparam ScanOp [inferred] Binary scan operator type having member T operator()(const T &a, const T &b) + */ + template + __device__ __forceinline__ void ExclusiveScan( + T input, ///< [in] Calling thread's input items + T &output, ///< [out] Calling thread's output items (may be aliased to \p input) + const T &identity, ///< [in] Identity value + ScanOp scan_op, ///< [in] Binary scan operator + T &block_aggregate) ///< [out] block-wide aggregate reduction of input items + { + InternalBlockScan(temp_storage, linear_tid).ExclusiveScan(input, output, identity, scan_op, block_aggregate); + } + + + /** + * \brief Computes an exclusive block-wide prefix scan using the specified binary \p scan_op functor. Each thread contributes one input element. the call-back functor \p block_prefix_op is invoked by the first warp in the block, and the value returned by lane0 in that warp is used as the "seed" value that logically prefixes the threadblock's scan inputs. Also provides every thread with the block-wide \p block_aggregate of all inputs. + * + * The \p block_prefix_op functor must implement a member function T operator()(T block_aggregate). + * The functor's input parameter \p block_aggregate is the same value also returned by the scan operation. + * The functor will be invoked by the first warp of threads in the block, however only the return value from + * lane0 is applied as the block-wide prefix. Can be stateful. + * + * Supports non-commutative scan operators. + * + * \blocked + * + * \smemreuse + * + * The code snippet below illustrates a single thread block that progressively + * computes an exclusive prefix max scan over multiple "tiles" of input using a + * prefix functor to maintain a running total between block-wide scans. Each tile consists + * of 128 integer items that are partitioned across 128 threads. + * \par + * \code + * #include + * + * // A stateful callback functor that maintains a running prefix to be applied + * // during consecutive scan operations. + * struct BlockPrefixOp + * { + * // Running prefix + * int running_total; + * + * // Constructor + * __device__ BlockPrefixOp(int running_total) : running_total(running_total) {} + * + * // Callback operator to be entered by the first warp of threads in the block. + * // Thread-0 is responsible for returning a value for seeding the block-wide scan. + * __device__ int operator()(int block_aggregate) + * { + * int old_prefix = running_total; + * running_total = (block_aggregate > old_prefix) ? block_aggregate : old_prefix; + * return old_prefix; + * } + * }; + * + * __global__ void ExampleKernel(int *d_data, int num_items, ...) + * { + * // Specialize BlockScan for 128 threads + * typedef cub::BlockScan BlockScan; + * + * // Allocate shared memory for BlockScan + * __shared__ typename BlockScan::TempStorage temp_storage; + * + * // Initialize running total + * BlockPrefixOp prefix_op(INT_MIN); + * + * // Have the block iterate over segments of items + * for (int block_offset = 0; block_offset < num_items; block_offset += 128) + * { + * // Load a segment of consecutive items that are blocked across threads + * int thread_data = d_data[block_offset]; + * + * // Collectively compute the block-wide exclusive prefix max scan + * int block_aggregate; + * BlockScan(temp_storage).ExclusiveScan( + * thread_data, thread_data, INT_MIN, cub::Max(), block_aggregate, prefix_op); + * __syncthreads(); + * + * // Store scanned items to output segment + * d_data[block_offset] = thread_data; + * } + * \endcode + * \par + * Suppose the input \p d_data is 0, -1, 2, -3, 4, -5, .... + * The corresponding output for the first segment will be INT_MIN, 0, 0, 2, ..., 124, 126. + * The output for the second segment will be 126, 128, 128, 130, ..., 252, 254. Furthermore, + * \p block_aggregate will be assigned \p 126 in all threads after the first scan, assigned \p 254 after the second + * scan, etc. + * + * \tparam ScanOp [inferred] Binary scan operator type having member T operator()(const T &a, const T &b) + * \tparam BlockPrefixOp [inferred] Call-back functor type having member T operator()(T block_aggregate) + */ + template < + typename ScanOp, + typename BlockPrefixOp> + __device__ __forceinline__ void ExclusiveScan( + T input, ///< [in] Calling thread's input item + T &output, ///< [out] Calling thread's output item (may be aliased to \p input) + T identity, ///< [in] Identity value + ScanOp scan_op, ///< [in] Binary scan operator + T &block_aggregate, ///< [out] block-wide aggregate reduction of input items (exclusive of the \p block_prefix_op value) + BlockPrefixOp &block_prefix_op) ///< [in-out] [warp0 only] Call-back functor for specifying a block-wide prefix to be applied to all inputs. + { + InternalBlockScan(temp_storage, linear_tid).ExclusiveScan(input, output, identity, scan_op, block_aggregate, block_prefix_op); + } + + + //@} end member group // Inclusive prefix sums + /******************************************************************//** + * \name Exclusive prefix scan operations (multiple data per thread) + *********************************************************************/ + //@{ + + + /** + * \brief Computes an exclusive block-wide prefix scan using the specified binary \p scan_op functor. Each thread contributes an array of consecutive input elements. + * + * Supports non-commutative scan operators. + * + * \blocked + * + * \smemreuse + * + * The code snippet below illustrates an exclusive prefix max scan of 512 integer items that + * are partitioned in a [blocked arrangement](index.html#sec5sec4) across 128 threads + * where each thread owns 4 consecutive items. + * \par + * \code + * #include + * + * __global__ void ExampleKernel(...) + * { + * // Specialize BlockScan for 128 threads on type int + * typedef cub::BlockScan BlockScan; + * + * // Allocate shared memory for BlockScan + * __shared__ typename BlockScan::TempStorage temp_storage; + * + * // Obtain a segment of consecutive items that are blocked across threads + * int thread_data[4]; + * ... + * + * // Collectively compute the block-wide exclusive prefix max scan + * BlockScan(temp_storage).ExclusiveScan(thread_data, thread_data, INT_MIN, cub::Max()); + * + * \endcode + * \par + * Suppose the set of input \p thread_data across the block of threads is + * { [0,-1,2,-3], [4,-5,6,-7], ..., [508,-509,510,-511] }. + * The corresponding output \p thread_data in those threads will be + * { [INT_MIN,0,0,2], [2,4,4,6], ..., [506,508,508,510] }. + * + * \tparam ITEMS_PER_THREAD [inferred] The number of consecutive items partitioned onto each thread. + * \tparam ScanOp [inferred] Binary scan operator type having member T operator()(const T &a, const T &b) + */ + template < + int ITEMS_PER_THREAD, + typename ScanOp> + __device__ __forceinline__ void ExclusiveScan( + T (&input)[ITEMS_PER_THREAD], ///< [in] Calling thread's input items + T (&output)[ITEMS_PER_THREAD], ///< [out] Calling thread's output items (may be aliased to \p input) + const T &identity, ///< [in] Identity value + ScanOp scan_op) ///< [in] Binary scan operator + { + // Reduce consecutive thread items in registers + T thread_partial = ThreadReduce(input, scan_op); + + // Exclusive threadblock-scan + ExclusiveScan(thread_partial, thread_partial, identity, scan_op); + + // Exclusive scan in registers with prefix + ThreadScanExclusive(input, output, scan_op, thread_partial); + } + + + /** + * \brief Computes an exclusive block-wide prefix scan using the specified binary \p scan_op functor. Each thread contributes an array of consecutive input elements. Also provides every thread with the block-wide \p block_aggregate of all inputs. + * + * Supports non-commutative scan operators. + * + * \blocked + * + * \smemreuse + * + * The code snippet below illustrates an exclusive prefix max scan of 512 integer items that + * are partitioned in a [blocked arrangement](index.html#sec5sec4) across 128 threads + * where each thread owns 4 consecutive items. + * \par + * \code + * #include + * + * __global__ void ExampleKernel(...) + * { + * // Specialize BlockScan for 128 threads on type int + * typedef cub::BlockScan BlockScan; + * + * // Allocate shared memory for BlockScan + * __shared__ typename BlockScan::TempStorage temp_storage; + * + * // Obtain a segment of consecutive items that are blocked across threads + * int thread_data[4]; + * ... + * + * // Collectively compute the block-wide exclusive prefix max scan + * int block_aggregate; + * BlockScan(temp_storage).ExclusiveScan(thread_data, thread_data, INT_MIN, cub::Max(), block_aggregate); + * + * \endcode + * \par + * Suppose the set of input \p thread_data across the block of threads is { [0,-1,2,-3], [4,-5,6,-7], ..., [508,-509,510,-511] }. The + * corresponding output \p thread_data in those threads will be { [INT_MIN,0,0,2], [2,4,4,6], ..., [506,508,508,510] }. + * Furthermore the value \p 510 will be stored in \p block_aggregate for all threads. + * + * \tparam ITEMS_PER_THREAD [inferred] The number of consecutive items partitioned onto each thread. + * \tparam ScanOp [inferred] Binary scan operator type having member T operator()(const T &a, const T &b) + */ + template < + int ITEMS_PER_THREAD, + typename ScanOp> + __device__ __forceinline__ void ExclusiveScan( + T (&input)[ITEMS_PER_THREAD], ///< [in] Calling thread's input items + T (&output)[ITEMS_PER_THREAD], ///< [out] Calling thread's output items (may be aliased to \p input) + const T &identity, ///< [in] Identity value + ScanOp scan_op, ///< [in] Binary scan operator + T &block_aggregate) ///< [out] block-wide aggregate reduction of input items + { + // Reduce consecutive thread items in registers + T thread_partial = ThreadReduce(input, scan_op); + + // Exclusive threadblock-scan + ExclusiveScan(thread_partial, thread_partial, identity, scan_op, block_aggregate); + + // Exclusive scan in registers with prefix + ThreadScanExclusive(input, output, scan_op, thread_partial); + } + + + /** + * \brief Computes an exclusive block-wide prefix scan using the specified binary \p scan_op functor. Each thread contributes an array of consecutive input elements. the call-back functor \p block_prefix_op is invoked by the first warp in the block, and the value returned by lane0 in that warp is used as the "seed" value that logically prefixes the threadblock's scan inputs. Also provides every thread with the block-wide \p block_aggregate of all inputs. + * + * The \p block_prefix_op functor must implement a member function T operator()(T block_aggregate). + * The functor's input parameter \p block_aggregate is the same value also returned by the scan operation. + * The functor will be invoked by the first warp of threads in the block, however only the return value from + * lane0 is applied as the block-wide prefix. Can be stateful. + * + * Supports non-commutative scan operators. + * + * \blocked + * + * \smemreuse + * + * The code snippet below illustrates a single thread block that progressively + * computes an exclusive prefix max scan over multiple "tiles" of input using a + * prefix functor to maintain a running total between block-wide scans. Each tile consists + * of 128 integer items that are partitioned across 128 threads. + * \par + * \code + * #include + * + * // A stateful callback functor that maintains a running prefix to be applied + * // during consecutive scan operations. + * struct BlockPrefixOp + * { + * // Running prefix + * int running_total; + * + * // Constructor + * __device__ BlockPrefixOp(int running_total) : running_total(running_total) {} + * + * // Callback operator to be entered by the first warp of threads in the block. + * // Thread-0 is responsible for returning a value for seeding the block-wide scan. + * __device__ int operator()(int block_aggregate) + * { + * int old_prefix = running_total; + * running_total = (block_aggregate > old_prefix) ? block_aggregate : old_prefix; + * return old_prefix; + * } + * }; + * + * __global__ void ExampleKernel(int *d_data, int num_items, ...) + * { + * // Specialize BlockLoad, BlockStore, and BlockScan for 128 threads, 4 ints per thread + * typedef cub::BlockLoad BlockLoad; + * typedef cub::BlockStore BlockStore; + * typedef cub::BlockScan BlockScan; + * + * // Allocate aliased shared memory for BlockLoad, BlockStore, and BlockScan + * __shared__ union { + * typename BlockLoad::TempStorage load; + * typename BlockScan::TempStorage scan; + * typename BlockStore::TempStorage store; + * } temp_storage; + * + * // Initialize running total + * BlockPrefixOp prefix_op(0); + * + * // Have the block iterate over segments of items + * for (int block_offset = 0; block_offset < num_items; block_offset += 128 * 4) + * { + * // Load a segment of consecutive items that are blocked across threads + * int thread_data[4]; + * BlockLoad(temp_storage.load).Load(d_data + block_offset, thread_data); + * __syncthreads(); + * + * // Collectively compute the block-wide exclusive prefix max scan + * int block_aggregate; + * BlockScan(temp_storage.scan).ExclusiveScan( + * thread_data, thread_data, INT_MIN, cub::Max(), block_aggregate, prefix_op); + * __syncthreads(); + * + * // Store scanned items to output segment + * BlockStore(temp_storage.store).Store(d_data + block_offset, thread_data); + * __syncthreads(); + * } + * \endcode + * \par + * Suppose the input \p d_data is 0, -1, 2, -3, 4, -5, .... + * The corresponding output for the first segment will be INT_MIN, 0, 0, 2, 2, 4, ..., 508, 510. + * The output for the second segment will be 510, 512, 512, 514, 514, 516, ..., 1020, 1022. Furthermore, + * \p block_aggregate will be assigned \p 510 in all threads after the first scan, assigned \p 1022 after the second + * scan, etc. + * + * \tparam ITEMS_PER_THREAD [inferred] The number of consecutive items partitioned onto each thread. + * \tparam ScanOp [inferred] Binary scan operator type having member T operator()(const T &a, const T &b) + * \tparam BlockPrefixOp [inferred] Call-back functor type having member T operator()(T block_aggregate) + */ + template < + int ITEMS_PER_THREAD, + typename ScanOp, + typename BlockPrefixOp> + __device__ __forceinline__ void ExclusiveScan( + T (&input)[ITEMS_PER_THREAD], ///< [in] Calling thread's input items + T (&output)[ITEMS_PER_THREAD], ///< [out] Calling thread's output items (may be aliased to \p input) + T identity, ///< [in] Identity value + ScanOp scan_op, ///< [in] Binary scan operator + T &block_aggregate, ///< [out] block-wide aggregate reduction of input items (exclusive of the \p block_prefix_op value) + BlockPrefixOp &block_prefix_op) ///< [in-out] [warp0 only] Call-back functor for specifying a block-wide prefix to be applied to all inputs. + { + // Reduce consecutive thread items in registers + T thread_partial = ThreadReduce(input, scan_op); + + // Exclusive threadblock-scan + ExclusiveScan(thread_partial, thread_partial, identity, scan_op, block_aggregate, block_prefix_op); + + // Exclusive scan in registers with prefix + ThreadScanExclusive(input, output, scan_op, thread_partial); + } + + + //@} end member group + +#ifndef DOXYGEN_SHOULD_SKIP_THIS // Do not document + + /******************************************************************//** + * \name Exclusive prefix scan operations (identityless, single datum per thread) + *********************************************************************/ + //@{ + + + /** + * \brief Computes an exclusive block-wide prefix scan using the specified binary \p scan_op functor. Each thread contributes one input element. With no identity value, the output computed for thread0 is undefined. + * + * Supports non-commutative scan operators. + * + * \blocked + * + * \smemreuse + * + * \tparam ScanOp [inferred] Binary scan operator type having member T operator()(const T &a, const T &b) + */ + template + __device__ __forceinline__ void ExclusiveScan( + T input, ///< [in] Calling thread's input item + T &output, ///< [out] Calling thread's output item (may be aliased to \p input) + ScanOp scan_op) ///< [in] Binary scan operator + { + T block_aggregate; + InternalBlockScan(temp_storage, linear_tid).ExclusiveScan(input, output, scan_op, block_aggregate); + } + + + /** + * \brief Computes an exclusive block-wide prefix scan using the specified binary \p scan_op functor. Each thread contributes one input element. Also provides every thread with the block-wide \p block_aggregate of all inputs. With no identity value, the output computed for thread0 is undefined. + * + * Supports non-commutative scan operators. + * + * \blocked + * + * \smemreuse + * + * \tparam ScanOp [inferred] Binary scan operator type having member T operator()(const T &a, const T &b) + */ + template + __device__ __forceinline__ void ExclusiveScan( + T input, ///< [in] Calling thread's input item + T &output, ///< [out] Calling thread's output item (may be aliased to \p input) + ScanOp scan_op, ///< [in] Binary scan operator + T &block_aggregate) ///< [out] block-wide aggregate reduction of input items + { + InternalBlockScan(temp_storage, linear_tid).ExclusiveScan(input, output, scan_op, block_aggregate); + } + + + /** + * \brief Computes an exclusive block-wide prefix scan using the specified binary \p scan_op functor. Each thread contributes one input element. the call-back functor \p block_prefix_op is invoked by the first warp in the block, and the value returned by lane0 in that warp is used as the "seed" value that logically prefixes the threadblock's scan inputs. Also provides every thread with the block-wide \p block_aggregate of all inputs. + * + * The \p block_prefix_op functor must implement a member function T operator()(T block_aggregate). + * The functor's input parameter \p block_aggregate is the same value also returned by the scan operation. + * The functor will be invoked by the first warp of threads in the block, however only the return value from + * lane0 is applied as the block-wide prefix. Can be stateful. + * + * Supports non-commutative scan operators. + * + * \blocked + * + * \smemreuse + * + * \tparam ScanOp [inferred] Binary scan operator type having member T operator()(const T &a, const T &b) + * \tparam BlockPrefixOp [inferred] Call-back functor type having member T operator()(T block_aggregate) + */ + template < + typename ScanOp, + typename BlockPrefixOp> + __device__ __forceinline__ void ExclusiveScan( + T input, ///< [in] Calling thread's input item + T &output, ///< [out] Calling thread's output item (may be aliased to \p input) + ScanOp scan_op, ///< [in] Binary scan operator + T &block_aggregate, ///< [out] block-wide aggregate reduction of input items (exclusive of the \p block_prefix_op value) + BlockPrefixOp &block_prefix_op) ///< [in-out] [warp0 only] Call-back functor for specifying a block-wide prefix to be applied to all inputs. + { + InternalBlockScan(temp_storage, linear_tid).ExclusiveScan(input, output, scan_op, block_aggregate, block_prefix_op); + } + + + //@} end member group + /******************************************************************//** + * \name Exclusive prefix scan operations (identityless, multiple data per thread) + *********************************************************************/ + //@{ + + + /** + * \brief Computes an exclusive block-wide prefix scan using the specified binary \p scan_op functor. Each thread contributes an array of consecutive input elements. With no identity value, the output computed for thread0 is undefined. + * + * Supports non-commutative scan operators. + * + * \blocked + * + * \smemreuse + * + * \tparam ITEMS_PER_THREAD [inferred] The number of consecutive items partitioned onto each thread. + * \tparam ScanOp [inferred] Binary scan operator type having member T operator()(const T &a, const T &b) + */ + template < + int ITEMS_PER_THREAD, + typename ScanOp> + __device__ __forceinline__ void ExclusiveScan( + T (&input)[ITEMS_PER_THREAD], ///< [in] Calling thread's input items + T (&output)[ITEMS_PER_THREAD], ///< [out] Calling thread's output items (may be aliased to \p input) + ScanOp scan_op) ///< [in] Binary scan operator + { + // Reduce consecutive thread items in registers + T thread_partial = ThreadReduce(input, scan_op); + + // Exclusive threadblock-scan + ExclusiveScan(thread_partial, thread_partial, scan_op); + + // Exclusive scan in registers with prefix + ThreadScanExclusive(input, output, scan_op, thread_partial, (linear_tid != 0)); + } + + + /** + * \brief Computes an exclusive block-wide prefix scan using the specified binary \p scan_op functor. Each thread contributes an array of consecutive input elements. Also provides every thread with the block-wide \p block_aggregate of all inputs. With no identity value, the output computed for thread0 is undefined. + * + * Supports non-commutative scan operators. + * + * \blocked + * + * \smemreuse + * + * \tparam ITEMS_PER_THREAD [inferred] The number of consecutive items partitioned onto each thread. + * \tparam ScanOp [inferred] Binary scan operator type having member T operator()(const T &a, const T &b) + */ + template < + int ITEMS_PER_THREAD, + typename ScanOp> + __device__ __forceinline__ void ExclusiveScan( + T (&input)[ITEMS_PER_THREAD], ///< [in] Calling thread's input items + T (&output)[ITEMS_PER_THREAD], ///< [out] Calling thread's output items (may be aliased to \p input) + ScanOp scan_op, ///< [in] Binary scan operator + T &block_aggregate) ///< [out] block-wide aggregate reduction of input items + { + // Reduce consecutive thread items in registers + T thread_partial = ThreadReduce(input, scan_op); + + // Exclusive threadblock-scan + ExclusiveScan(thread_partial, thread_partial, scan_op, block_aggregate); + + // Exclusive scan in registers with prefix + ThreadScanExclusive(input, output, scan_op, thread_partial, (linear_tid != 0)); + } + + + /** + * \brief Computes an exclusive block-wide prefix scan using the specified binary \p scan_op functor. Each thread contributes an array of consecutive input elements. the call-back functor \p block_prefix_op is invoked by the first warp in the block, and the value returned by lane0 in that warp is used as the "seed" value that logically prefixes the threadblock's scan inputs. Also provides every thread with the block-wide \p block_aggregate of all inputs. + * + * The \p block_prefix_op functor must implement a member function T operator()(T block_aggregate). + * The functor's input parameter \p block_aggregate is the same value also returned by the scan operation. + * The functor will be invoked by the first warp of threads in the block, however only the return value from + * lane0 is applied as the block-wide prefix. Can be stateful. + * + * Supports non-commutative scan operators. + * + * \blocked + * + * \smemreuse + * + * \tparam ITEMS_PER_THREAD [inferred] The number of consecutive items partitioned onto each thread. + * \tparam ScanOp [inferred] Binary scan operator type having member T operator()(const T &a, const T &b) + * \tparam BlockPrefixOp [inferred] Call-back functor type having member T operator()(T block_aggregate) + */ + template < + int ITEMS_PER_THREAD, + typename ScanOp, + typename BlockPrefixOp> + __device__ __forceinline__ void ExclusiveScan( + T (&input)[ITEMS_PER_THREAD], ///< [in] Calling thread's input items + T (&output)[ITEMS_PER_THREAD], ///< [out] Calling thread's output items (may be aliased to \p input) + ScanOp scan_op, ///< [in] Binary scan operator + T &block_aggregate, ///< [out] block-wide aggregate reduction of input items (exclusive of the \p block_prefix_op value) + BlockPrefixOp &block_prefix_op) ///< [in-out] [warp0 only] Call-back functor for specifying a block-wide prefix to be applied to all inputs. + { + // Reduce consecutive thread items in registers + T thread_partial = ThreadReduce(input, scan_op); + + // Exclusive threadblock-scan + ExclusiveScan(thread_partial, thread_partial, scan_op, block_aggregate, block_prefix_op); + + // Exclusive scan in registers with prefix + ThreadScanExclusive(input, output, scan_op, thread_partial); + } + + + //@} end member group + +#endif // DOXYGEN_SHOULD_SKIP_THIS + + /******************************************************************//** + * \name Inclusive prefix sum operations + *********************************************************************/ + //@{ + + + /** + * \brief Computes an inclusive block-wide prefix scan using addition (+) as the scan operator. Each thread contributes one input element. + * + * \blocked + * + * \smemreuse + * + * The code snippet below illustrates an inclusive prefix sum of 128 integer items that + * are partitioned across 128 threads. + * \par + * \code + * #include + * + * __global__ void ExampleKernel(...) + * { + * // Specialize BlockScan for 128 threads on type int + * typedef cub::BlockScan BlockScan; + * + * // Allocate shared memory for BlockScan + * __shared__ typename BlockScan::TempStorage temp_storage; + * + * // Obtain input item for each thread + * int thread_data; + * ... + * + * // Collectively compute the block-wide inclusive prefix sum + * BlockScan(temp_storage).InclusiveSum(thread_data, thread_data); + * + * \endcode + * \par + * Suppose the set of input \p thread_data across the block of threads is 1, 1, ..., 1. The + * corresponding output \p thread_data in those threads will be 1, 2, ..., 128. + * + */ + __device__ __forceinline__ void InclusiveSum( + T input, ///< [in] Calling thread's input item + T &output) ///< [out] Calling thread's output item (may be aliased to \p input) + { + T block_aggregate; + InternalBlockScan(temp_storage, linear_tid).InclusiveSum(input, output, block_aggregate); + } + + + /** + * \brief Computes an inclusive block-wide prefix scan using addition (+) as the scan operator. Each thread contributes one input element. Also provides every thread with the block-wide \p block_aggregate of all inputs. + * + * \blocked + * + * \smemreuse + * + * The code snippet below illustrates an inclusive prefix sum of 128 integer items that + * are partitioned across 128 threads. + * \par + * \code + * #include + * + * __global__ void ExampleKernel(...) + * { + * // Specialize BlockScan for 128 threads on type int + * typedef cub::BlockScan BlockScan; + * + * // Allocate shared memory for BlockScan + * __shared__ typename BlockScan::TempStorage temp_storage; + * + * // Obtain input item for each thread + * int thread_data; + * ... + * + * // Collectively compute the block-wide inclusive prefix sum + * int block_aggregate; + * BlockScan(temp_storage).InclusiveSum(thread_data, thread_data, block_aggregate); + * + * \endcode + * \par + * Suppose the set of input \p thread_data across the block of threads is 1, 1, ..., 1. The + * corresponding output \p thread_data in those threads will be 1, 2, ..., 128. + * Furthermore the value \p 128 will be stored in \p block_aggregate for all threads. + * + */ + __device__ __forceinline__ void InclusiveSum( + T input, ///< [in] Calling thread's input item + T &output, ///< [out] Calling thread's output item (may be aliased to \p input) + T &block_aggregate) ///< [out] block-wide aggregate reduction of input items + { + InternalBlockScan(temp_storage, linear_tid).InclusiveSum(input, output, block_aggregate); + } + + + + /** + * \brief Computes an inclusive block-wide prefix scan using addition (+) as the scan operator. Each thread contributes one input element. Instead of using 0 as the block-wide prefix, the call-back functor \p block_prefix_op is invoked by the first warp in the block, and the value returned by lane0 in that warp is used as the "seed" value that logically prefixes the threadblock's scan inputs. Also provides every thread with the block-wide \p block_aggregate of all inputs. + * + * The \p block_prefix_op functor must implement a member function T operator()(T block_aggregate). + * The functor's input parameter \p block_aggregate is the same value also returned by the scan operation. + * The functor will be invoked by the first warp of threads in the block, however only the return value from + * lane0 is applied as the block-wide prefix. Can be stateful. + * + * \blocked + * + * \smemreuse + * + * The code snippet below illustrates a single thread block that progressively + * computes an inclusive prefix sum over multiple "tiles" of input using a + * prefix functor to maintain a running total between block-wide scans. Each tile consists + * of 128 integer items that are partitioned across 128 threads. + * \par + * \code + * #include + * + * // A stateful callback functor that maintains a running prefix to be applied + * // during consecutive scan operations. + * struct BlockPrefixOp + * { + * // Running prefix + * int running_total; + * + * // Constructor + * __device__ BlockPrefixOp(int running_total) : running_total(running_total) {} + * + * // Callback operator to be entered by the first warp of threads in the block. + * // Thread-0 is responsible for returning a value for seeding the block-wide scan. + * __device__ int operator()(int block_aggregate) + * { + * int old_prefix = running_total; + * running_total += block_aggregate; + * return old_prefix; + * } + * }; + * + * __global__ void ExampleKernel(int *d_data, int num_items, ...) + * { + * // Specialize BlockScan for 128 threads + * typedef cub::BlockScan BlockScan; + * + * // Allocate shared memory for BlockScan + * __shared__ typename BlockScan::TempStorage temp_storage; + * + * // Initialize running total + * BlockPrefixOp prefix_op(0); + * + * // Have the block iterate over segments of items + * for (int block_offset = 0; block_offset < num_items; block_offset += 128) + * { + * // Load a segment of consecutive items that are blocked across threads + * int thread_data = d_data[block_offset]; + * + * // Collectively compute the block-wide inclusive prefix sum + * int block_aggregate; + * BlockScan(temp_storage).InclusiveSum( + * thread_data, thread_data, block_aggregate, prefix_op); + * __syncthreads(); + * + * // Store scanned items to output segment + * d_data[block_offset] = thread_data; + * } + * \endcode + * \par + * Suppose the input \p d_data is 1, 1, 1, 1, 1, 1, 1, 1, .... + * The corresponding output for the first segment will be 1, 2, ..., 128. + * The output for the second segment will be 129, 130, ..., 256. Furthermore, + * the value \p 128 will be stored in \p block_aggregate for all threads after each scan. + * + * \tparam BlockPrefixOp [inferred] Call-back functor type having member T operator()(T block_aggregate) + */ + template + __device__ __forceinline__ void InclusiveSum( + T input, ///< [in] Calling thread's input item + T &output, ///< [out] Calling thread's output item (may be aliased to \p input) + T &block_aggregate, ///< [out] block-wide aggregate reduction of input items (exclusive of the \p block_prefix_op value) + BlockPrefixOp &block_prefix_op) ///< [in-out] [warp0 only] Call-back functor for specifying a block-wide prefix to be applied to all inputs. + { + InternalBlockScan(temp_storage, linear_tid).InclusiveSum(input, output, block_aggregate, block_prefix_op); + } + + + //@} end member group + /******************************************************************//** + * \name Inclusive prefix sum operations (multiple data per thread) + *********************************************************************/ + //@{ + + + /** + * \brief Computes an inclusive block-wide prefix scan using addition (+) as the scan operator. Each thread contributes an array of consecutive input elements. + * + * \blocked + * + * \smemreuse + * + * The code snippet below illustrates an inclusive prefix sum of 512 integer items that + * are partitioned in a [blocked arrangement](index.html#sec5sec4) across 128 threads + * where each thread owns 4 consecutive items. + * \par + * \code + * #include + * + * __global__ void ExampleKernel(...) + * { + * // Specialize BlockScan for 128 threads on type int + * typedef cub::BlockScan BlockScan; + * + * // Allocate shared memory for BlockScan + * __shared__ typename BlockScan::TempStorage temp_storage; + * + * // Obtain a segment of consecutive items that are blocked across threads + * int thread_data[4]; + * ... + * + * // Collectively compute the block-wide inclusive prefix sum + * BlockScan(temp_storage).InclusiveSum(thread_data, thread_data); + * + * \endcode + * \par + * Suppose the set of input \p thread_data across the block of threads is { [1,1,1,1], [1,1,1,1], ..., [1,1,1,1] }. The + * corresponding output \p thread_data in those threads will be { [1,2,3,4], [5,6,7,8], ..., [509,510,511,512] }. + * + * \tparam ITEMS_PER_THREAD [inferred] The number of consecutive items partitioned onto each thread. + */ + template + __device__ __forceinline__ void InclusiveSum( + T (&input)[ITEMS_PER_THREAD], ///< [in] Calling thread's input items + T (&output)[ITEMS_PER_THREAD]) ///< [out] Calling thread's output items (may be aliased to \p input) + { + if (ITEMS_PER_THREAD == 1) + { + InclusiveSum(input[0], output[0]); + } + else + { + // Reduce consecutive thread items in registers + Sum scan_op; + T thread_partial = ThreadReduce(input, scan_op); + + // Exclusive threadblock-scan + ExclusiveSum(thread_partial, thread_partial); + + // Inclusive scan in registers with prefix + ThreadScanInclusive(input, output, scan_op, thread_partial, (linear_tid != 0)); + } + } + + + /** + * \brief Computes an inclusive block-wide prefix scan using addition (+) as the scan operator. Each thread contributes an array of consecutive input elements. Also provides every thread with the block-wide \p block_aggregate of all inputs. + * + * \blocked + * + * \smemreuse + * + * The code snippet below illustrates an inclusive prefix sum of 512 integer items that + * are partitioned in a [blocked arrangement](index.html#sec5sec4) across 128 threads + * where each thread owns 4 consecutive items. + * \par + * \code + * #include + * + * __global__ void ExampleKernel(...) + * { + * // Specialize BlockScan for 128 threads on type int + * typedef cub::BlockScan BlockScan; + * + * // Allocate shared memory for BlockScan + * __shared__ typename BlockScan::TempStorage temp_storage; + * + * // Obtain a segment of consecutive items that are blocked across threads + * int thread_data[4]; + * ... + * + * // Collectively compute the block-wide inclusive prefix sum + * int block_aggregate; + * BlockScan(temp_storage).InclusiveSum(thread_data, thread_data, block_aggregate); + * + * \endcode + * \par + * Suppose the set of input \p thread_data across the block of threads is + * { [1,1,1,1], [1,1,1,1], ..., [1,1,1,1] }. The + * corresponding output \p thread_data in those threads will be + * { [1,2,3,4], [5,6,7,8], ..., [509,510,511,512] }. + * Furthermore the value \p 512 will be stored in \p block_aggregate for all threads. + * + * \tparam ITEMS_PER_THREAD [inferred] The number of consecutive items partitioned onto each thread. + * \tparam ScanOp [inferred] Binary scan operator type having member T operator()(const T &a, const T &b) + */ + template + __device__ __forceinline__ void InclusiveSum( + T (&input)[ITEMS_PER_THREAD], ///< [in] Calling thread's input items + T (&output)[ITEMS_PER_THREAD], ///< [out] Calling thread's output items (may be aliased to \p input) + T &block_aggregate) ///< [out] block-wide aggregate reduction of input items + { + if (ITEMS_PER_THREAD == 1) + { + InclusiveSum(input[0], output[0], block_aggregate); + } + else + { + // Reduce consecutive thread items in registers + Sum scan_op; + T thread_partial = ThreadReduce(input, scan_op); + + // Exclusive threadblock-scan + ExclusiveSum(thread_partial, thread_partial, block_aggregate); + + // Inclusive scan in registers with prefix + ThreadScanInclusive(input, output, scan_op, thread_partial, (linear_tid != 0)); + } + } + + + /** + * \brief Computes an inclusive block-wide prefix scan using addition (+) as the scan operator. Each thread contributes an array of consecutive input elements. Instead of using 0 as the block-wide prefix, the call-back functor \p block_prefix_op is invoked by the first warp in the block, and the value returned by lane0 in that warp is used as the "seed" value that logically prefixes the threadblock's scan inputs. Also provides every thread with the block-wide \p block_aggregate of all inputs. + * + * The \p block_prefix_op functor must implement a member function T operator()(T block_aggregate). + * The functor's input parameter \p block_aggregate is the same value also returned by the scan operation. + * The functor will be invoked by the first warp of threads in the block, however only the return value from + * lane0 is applied as the block-wide prefix. Can be stateful. + * + * \blocked + * + * \smemreuse + * + * The code snippet below illustrates a single thread block that progressively + * computes an inclusive prefix sum over multiple "tiles" of input using a + * prefix functor to maintain a running total between block-wide scans. Each tile consists + * of 512 integer items that are partitioned in a [blocked arrangement](index.html#sec5sec4) + * across 128 threads where each thread owns 4 consecutive items. + * \par + * \code + * #include + * + * // A stateful callback functor that maintains a running prefix to be applied + * // during consecutive scan operations. + * struct BlockPrefixOp + * { + * // Running prefix + * int running_total; + * + * // Constructor + * __device__ BlockPrefixOp(int running_total) : running_total(running_total) {} + * + * // Callback operator to be entered by the first warp of threads in the block. + * // Thread-0 is responsible for returning a value for seeding the block-wide scan. + * __device__ int operator()(int block_aggregate) + * { + * int old_prefix = running_total; + * running_total += block_aggregate; + * return old_prefix; + * } + * }; + * + * __global__ void ExampleKernel(int *d_data, int num_items, ...) + * { + * // Specialize BlockLoad, BlockStore, and BlockScan for 128 threads, 4 ints per thread + * typedef cub::BlockLoad BlockLoad; + * typedef cub::BlockStore BlockStore; + * typedef cub::BlockScan BlockScan; + * + * // Allocate aliased shared memory for BlockLoad, BlockStore, and BlockScan + * __shared__ union { + * typename BlockLoad::TempStorage load; + * typename BlockScan::TempStorage scan; + * typename BlockStore::TempStorage store; + * } temp_storage; + * + * // Initialize running total + * BlockPrefixOp prefix_op(0); + * + * // Have the block iterate over segments of items + * for (int block_offset = 0; block_offset < num_items; block_offset += 128 * 4) + * { + * // Load a segment of consecutive items that are blocked across threads + * int thread_data[4]; + * BlockLoad(temp_storage.load).Load(d_data + block_offset, thread_data); + * __syncthreads(); + * + * // Collectively compute the block-wide inclusive prefix sum + * int block_aggregate; + * BlockScan(temp_storage.scan).IncluisveSum( + * thread_data, thread_data, block_aggregate, prefix_op); + * __syncthreads(); + * + * // Store scanned items to output segment + * BlockStore(temp_storage.store).Store(d_data + block_offset, thread_data); + * __syncthreads(); + * } + * \endcode + * \par + * Suppose the input \p d_data is 1, 1, 1, 1, 1, 1, 1, 1, .... + * The corresponding output for the first segment will be 1, 2, 3, 4, ..., 511, 512. + * The output for the second segment will be 513, 514, 515, 516, ..., 1023, 1024. Furthermore, + * the value \p 512 will be stored in \p block_aggregate for all threads after each scan. + * + * \tparam ITEMS_PER_THREAD [inferred] The number of consecutive items partitioned onto each thread. + * \tparam BlockPrefixOp [inferred] Call-back functor type having member T operator()(T block_aggregate) + */ + template < + int ITEMS_PER_THREAD, + typename BlockPrefixOp> + __device__ __forceinline__ void InclusiveSum( + T (&input)[ITEMS_PER_THREAD], ///< [in] Calling thread's input items + T (&output)[ITEMS_PER_THREAD], ///< [out] Calling thread's output items (may be aliased to \p input) + T &block_aggregate, ///< [out] block-wide aggregate reduction of input items (exclusive of the \p block_prefix_op value) + BlockPrefixOp &block_prefix_op) ///< [in-out] [warp0 only] Call-back functor for specifying a block-wide prefix to be applied to all inputs. + { + if (ITEMS_PER_THREAD == 1) + { + InclusiveSum(input[0], output[0], block_aggregate, block_prefix_op); + } + else + { + // Reduce consecutive thread items in registers + Sum scan_op; + T thread_partial = ThreadReduce(input, scan_op); + + // Exclusive threadblock-scan + ExclusiveSum(thread_partial, thread_partial, block_aggregate, block_prefix_op); + + // Inclusive scan in registers with prefix + ThreadScanInclusive(input, output, scan_op, thread_partial); + } + } + + + //@} end member group + /******************************************************************//** + * \name Inclusive prefix scan operations + *********************************************************************/ + //@{ + + + /** + * \brief Computes an inclusive block-wide prefix scan using the specified binary \p scan_op functor. Each thread contributes one input element. + * + * Supports non-commutative scan operators. + * + * \blocked + * + * \smemreuse + * + * The code snippet below illustrates an inclusive prefix max scan of 128 integer items that + * are partitioned across 128 threads. + * \par + * \code + * #include + * + * __global__ void ExampleKernel(...) + * { + * // Specialize BlockScan for 128 threads on type int + * typedef cub::BlockScan BlockScan; + * + * // Allocate shared memory for BlockScan + * __shared__ typename BlockScan::TempStorage temp_storage; + * + * // Obtain input item for each thread + * int thread_data; + * ... + * + * // Collectively compute the block-wide inclusive prefix max scan + * BlockScan(temp_storage).InclusiveScan(thread_data, thread_data, cub::Max()); + * + * \endcode + * \par + * Suppose the set of input \p thread_data across the block of threads is 0, -1, 2, -3, ..., 126, -127. The + * corresponding output \p thread_data in those threads will be 0, 0, 2, 2, ..., 126, 126. + * + * \tparam ScanOp [inferred] Binary scan operator type having member T operator()(const T &a, const T &b) + */ + template + __device__ __forceinline__ void InclusiveScan( + T input, ///< [in] Calling thread's input item + T &output, ///< [out] Calling thread's output item (may be aliased to \p input) + ScanOp scan_op) ///< [in] Binary scan operator + { + T block_aggregate; + InclusiveScan(input, output, scan_op, block_aggregate); + } + + + /** + * \brief Computes an inclusive block-wide prefix scan using the specified binary \p scan_op functor. Each thread contributes one input element. Also provides every thread with the block-wide \p block_aggregate of all inputs. + * + * Supports non-commutative scan operators. + * + * \blocked + * + * \smemreuse + * + * The code snippet below illustrates an inclusive prefix max scan of 128 integer items that + * are partitioned across 128 threads. + * \par + * \code + * #include + * + * __global__ void ExampleKernel(...) + * { + * // Specialize BlockScan for 128 threads on type int + * typedef cub::BlockScan BlockScan; + * + * // Allocate shared memory for BlockScan + * __shared__ typename BlockScan::TempStorage temp_storage; + * + * // Obtain input item for each thread + * int thread_data; + * ... + * + * // Collectively compute the block-wide inclusive prefix max scan + * int block_aggregate; + * BlockScan(temp_storage).InclusiveScan(thread_data, thread_data, cub::Max(), block_aggregate); + * + * \endcode + * \par + * Suppose the set of input \p thread_data across the block of threads is 0, -1, 2, -3, ..., 126, -127. The + * corresponding output \p thread_data in those threads will be 0, 0, 2, 2, ..., 126, 126. + * Furthermore the value \p 126 will be stored in \p block_aggregate for all threads. + * + * \tparam ScanOp [inferred] Binary scan operator type having member T operator()(const T &a, const T &b) + */ + template + __device__ __forceinline__ void InclusiveScan( + T input, ///< [in] Calling thread's input item + T &output, ///< [out] Calling thread's output item (may be aliased to \p input) + ScanOp scan_op, ///< [in] Binary scan operator + T &block_aggregate) ///< [out] block-wide aggregate reduction of input items + { + InternalBlockScan(temp_storage, linear_tid).InclusiveScan(input, output, scan_op, block_aggregate); + } + + + /** + * \brief Computes an inclusive block-wide prefix scan using the specified binary \p scan_op functor. Each thread contributes one input element. the call-back functor \p block_prefix_op is invoked by the first warp in the block, and the value returned by lane0 in that warp is used as the "seed" value that logically prefixes the threadblock's scan inputs. Also provides every thread with the block-wide \p block_aggregate of all inputs. + * + * The \p block_prefix_op functor must implement a member function T operator()(T block_aggregate). + * The functor's input parameter \p block_aggregate is the same value also returned by the scan operation. + * The functor will be invoked by the first warp of threads in the block, however only the return value from + * lane0 is applied as the block-wide prefix. Can be stateful. + * + * Supports non-commutative scan operators. + * + * \blocked + * + * \smemreuse + * + * The code snippet below illustrates a single thread block that progressively + * computes an inclusive prefix max scan over multiple "tiles" of input using a + * prefix functor to maintain a running total between block-wide scans. Each tile consists + * of 128 integer items that are partitioned across 128 threads. + * \par + * \code + * #include + * + * // A stateful callback functor that maintains a running prefix to be applied + * // during consecutive scan operations. + * struct BlockPrefixOp + * { + * // Running prefix + * int running_total; + * + * // Constructor + * __device__ BlockPrefixOp(int running_total) : running_total(running_total) {} + * + * // Callback operator to be entered by the first warp of threads in the block. + * // Thread-0 is responsible for returning a value for seeding the block-wide scan. + * __device__ int operator()(int block_aggregate) + * { + * int old_prefix = running_total; + * running_total = (block_aggregate > old_prefix) ? block_aggregate : old_prefix; + * return old_prefix; + * } + * }; + * + * __global__ void ExampleKernel(int *d_data, int num_items, ...) + * { + * // Specialize BlockScan for 128 threads + * typedef cub::BlockScan BlockScan; + * + * // Allocate shared memory for BlockScan + * __shared__ typename BlockScan::TempStorage temp_storage; + * + * // Initialize running total + * BlockPrefixOp prefix_op(INT_MIN); + * + * // Have the block iterate over segments of items + * for (int block_offset = 0; block_offset < num_items; block_offset += 128) + * { + * // Load a segment of consecutive items that are blocked across threads + * int thread_data = d_data[block_offset]; + * + * // Collectively compute the block-wide inclusive prefix max scan + * int block_aggregate; + * BlockScan(temp_storage).InclusiveScan( + * thread_data, thread_data, cub::Max(), block_aggregate, prefix_op); + * __syncthreads(); + * + * // Store scanned items to output segment + * d_data[block_offset] = thread_data; + * } + * \endcode + * \par + * Suppose the input \p d_data is 0, -1, 2, -3, 4, -5, .... + * The corresponding output for the first segment will be 0, 0, 2, 2, ..., 126, 126. + * The output for the second segment will be 128, 128, 130, 130, ..., 254, 254. Furthermore, + * \p block_aggregate will be assigned \p 126 in all threads after the first scan, assigned \p 254 after the second + * scan, etc. + * + * \tparam ScanOp [inferred] Binary scan operator type having member T operator()(const T &a, const T &b) + * \tparam BlockPrefixOp [inferred] Call-back functor type having member T operator()(T block_aggregate) + */ + template < + typename ScanOp, + typename BlockPrefixOp> + __device__ __forceinline__ void InclusiveScan( + T input, ///< [in] Calling thread's input item + T &output, ///< [out] Calling thread's output item (may be aliased to \p input) + ScanOp scan_op, ///< [in] Binary scan operator + T &block_aggregate, ///< [out] block-wide aggregate reduction of input items (exclusive of the \p block_prefix_op value) + BlockPrefixOp &block_prefix_op) ///< [in-out] [warp0 only] Call-back functor for specifying a block-wide prefix to be applied to all inputs. + { + InternalBlockScan(temp_storage, linear_tid).InclusiveScan(input, output, scan_op, block_aggregate, block_prefix_op); + } + + + //@} end member group + /******************************************************************//** + * \name Inclusive prefix scan operations (multiple data per thread) + *********************************************************************/ + //@{ + + + /** + * \brief Computes an inclusive block-wide prefix scan using the specified binary \p scan_op functor. Each thread contributes an array of consecutive input elements. + * + * Supports non-commutative scan operators. + * + * \blocked + * + * \smemreuse + * + * The code snippet below illustrates an inclusive prefix max scan of 512 integer items that + * are partitioned in a [blocked arrangement](index.html#sec5sec4) across 128 threads + * where each thread owns 4 consecutive items. + * \par + * \code + * #include + * + * __global__ void ExampleKernel(...) + * { + * // Specialize BlockScan for 128 threads on type int + * typedef cub::BlockScan BlockScan; + * + * // Allocate shared memory for BlockScan + * __shared__ typename BlockScan::TempStorage temp_storage; + * + * // Obtain a segment of consecutive items that are blocked across threads + * int thread_data[4]; + * ... + * + * // Collectively compute the block-wide inclusive prefix max scan + * BlockScan(temp_storage).InclusiveScan(thread_data, thread_data, cub::Max()); + * + * \endcode + * \par + * Suppose the set of input \p thread_data across the block of threads is { [0,-1,2,-3], [4,-5,6,-7], ..., [508,-509,510,-511] }. The + * corresponding output \p thread_data in those threads will be { [0,0,2,2], [4,4,6,6], ..., [508,508,510,510] }. + * + * \tparam ITEMS_PER_THREAD [inferred] The number of consecutive items partitioned onto each thread. + * \tparam ScanOp [inferred] Binary scan operator type having member T operator()(const T &a, const T &b) + */ + template < + int ITEMS_PER_THREAD, + typename ScanOp> + __device__ __forceinline__ void InclusiveScan( + T (&input)[ITEMS_PER_THREAD], ///< [in] Calling thread's input items + T (&output)[ITEMS_PER_THREAD], ///< [out] Calling thread's output items (may be aliased to \p input) + ScanOp scan_op) ///< [in] Binary scan operator + { + if (ITEMS_PER_THREAD == 1) + { + InclusiveScan(input[0], output[0], scan_op); + } + else + { + // Reduce consecutive thread items in registers + T thread_partial = ThreadReduce(input, scan_op); + + // Exclusive threadblock-scan + ExclusiveScan(thread_partial, thread_partial, scan_op); + + // Inclusive scan in registers with prefix + ThreadScanInclusive(input, output, scan_op, thread_partial, (linear_tid != 0)); + } + } + + + /** + * \brief Computes an inclusive block-wide prefix scan using the specified binary \p scan_op functor. Each thread contributes an array of consecutive input elements. Also provides every thread with the block-wide \p block_aggregate of all inputs. + * + * Supports non-commutative scan operators. + * + * \blocked + * + * \smemreuse + * + * The code snippet below illustrates an inclusive prefix max scan of 512 integer items that + * are partitioned in a [blocked arrangement](index.html#sec5sec4) across 128 threads + * where each thread owns 4 consecutive items. + * \par + * \code + * #include + * + * __global__ void ExampleKernel(...) + * { + * // Specialize BlockScan for 128 threads on type int + * typedef cub::BlockScan BlockScan; + * + * // Allocate shared memory for BlockScan + * __shared__ typename BlockScan::TempStorage temp_storage; + * + * // Obtain a segment of consecutive items that are blocked across threads + * int thread_data[4]; + * ... + * + * // Collectively compute the block-wide inclusive prefix max scan + * int block_aggregate; + * BlockScan(temp_storage).InclusiveScan(thread_data, thread_data, cub::Max(), block_aggregate); + * + * \endcode + * \par + * Suppose the set of input \p thread_data across the block of threads is + * { [0,-1,2,-3], [4,-5,6,-7], ..., [508,-509,510,-511] }. + * The corresponding output \p thread_data in those threads will be + * { [0,0,2,2], [4,4,6,6], ..., [508,508,510,510] }. + * Furthermore the value \p 510 will be stored in \p block_aggregate for all threads. + * + * \tparam ITEMS_PER_THREAD [inferred] The number of consecutive items partitioned onto each thread. + * \tparam ScanOp [inferred] Binary scan operator type having member T operator()(const T &a, const T &b) + */ + template < + int ITEMS_PER_THREAD, + typename ScanOp> + __device__ __forceinline__ void InclusiveScan( + T (&input)[ITEMS_PER_THREAD], ///< [in] Calling thread's input items + T (&output)[ITEMS_PER_THREAD], ///< [out] Calling thread's output items (may be aliased to \p input) + ScanOp scan_op, ///< [in] Binary scan operator + T &block_aggregate) ///< [out] block-wide aggregate reduction of input items + { + if (ITEMS_PER_THREAD == 1) + { + InclusiveScan(input[0], output[0], scan_op, block_aggregate); + } + else + { + // Reduce consecutive thread items in registers + T thread_partial = ThreadReduce(input, scan_op); + + // Exclusive threadblock-scan + ExclusiveScan(thread_partial, thread_partial, scan_op, block_aggregate); + + // Inclusive scan in registers with prefix + ThreadScanInclusive(input, output, scan_op, thread_partial, (linear_tid != 0)); + } + } + + + /** + * \brief Computes an inclusive block-wide prefix scan using the specified binary \p scan_op functor. Each thread contributes an array of consecutive input elements. the call-back functor \p block_prefix_op is invoked by the first warp in the block, and the value returned by lane0 in that warp is used as the "seed" value that logically prefixes the threadblock's scan inputs. Also provides every thread with the block-wide \p block_aggregate of all inputs. + * + * The \p block_prefix_op functor must implement a member function T operator()(T block_aggregate). + * The functor's input parameter \p block_aggregate is the same value also returned by the scan operation. + * The functor will be invoked by the first warp of threads in the block, however only the return value from + * lane0 is applied as the block-wide prefix. Can be stateful. + * + * Supports non-commutative scan operators. + * + * \blocked + * + * \smemreuse + * + * The code snippet below illustrates a single thread block that progressively + * computes an inclusive prefix max scan over multiple "tiles" of input using a + * prefix functor to maintain a running total between block-wide scans. Each tile consists + * of 128 integer items that are partitioned across 128 threads. + * \par + * \code + * #include + * + * // A stateful callback functor that maintains a running prefix to be applied + * // during consecutive scan operations. + * struct BlockPrefixOp + * { + * // Running prefix + * int running_total; + * + * // Constructor + * __device__ BlockPrefixOp(int running_total) : running_total(running_total) {} + * + * // Callback operator to be entered by the first warp of threads in the block. + * // Thread-0 is responsible for returning a value for seeding the block-wide scan. + * __device__ int operator()(int block_aggregate) + * { + * int old_prefix = running_total; + * running_total = (block_aggregate > old_prefix) ? block_aggregate : old_prefix; + * return old_prefix; + * } + * }; + * + * __global__ void ExampleKernel(int *d_data, int num_items, ...) + * { + * // Specialize BlockLoad, BlockStore, and BlockScan for 128 threads, 4 ints per thread + * typedef cub::BlockLoad BlockLoad; + * typedef cub::BlockStore BlockStore; + * typedef cub::BlockScan BlockScan; + * + * // Allocate aliased shared memory for BlockLoad, BlockStore, and BlockScan + * __shared__ union { + * typename BlockLoad::TempStorage load; + * typename BlockScan::TempStorage scan; + * typename BlockStore::TempStorage store; + * } temp_storage; + * + * // Initialize running total + * BlockPrefixOp prefix_op(0); + * + * // Have the block iterate over segments of items + * for (int block_offset = 0; block_offset < num_items; block_offset += 128 * 4) + * { + * // Load a segment of consecutive items that are blocked across threads + * int thread_data[4]; + * BlockLoad(temp_storage.load).Load(d_data + block_offset, thread_data); + * __syncthreads(); + * + * // Collectively compute the block-wide inclusive prefix max scan + * int block_aggregate; + * BlockScan(temp_storage.scan).InclusiveScan( + * thread_data, thread_data, cub::Max(), block_aggregate, prefix_op); + * __syncthreads(); + * + * // Store scanned items to output segment + * BlockStore(temp_storage.store).Store(d_data + block_offset, thread_data); + * __syncthreads(); + * } + * \endcode + * \par + * Suppose the input \p d_data is 0, -1, 2, -3, 4, -5, .... + * The corresponding output for the first segment will be 0, 0, 2, 2, 4, 4, ..., 510, 510. + * The output for the second segment will be 512, 512, 514, 514, 516, 516, ..., 1022, 1022. Furthermore, + * \p block_aggregate will be assigned \p 510 in all threads after the first scan, assigned \p 1022 after the second + * scan, etc. + * + * \tparam ITEMS_PER_THREAD [inferred] The number of consecutive items partitioned onto each thread. + * \tparam ScanOp [inferred] Binary scan operator type having member T operator()(const T &a, const T &b) + * \tparam BlockPrefixOp [inferred] Call-back functor type having member T operator()(T block_aggregate) + */ + template < + int ITEMS_PER_THREAD, + typename ScanOp, + typename BlockPrefixOp> + __device__ __forceinline__ void InclusiveScan( + T (&input)[ITEMS_PER_THREAD], ///< [in] Calling thread's input items + T (&output)[ITEMS_PER_THREAD], ///< [out] Calling thread's output items (may be aliased to \p input) + ScanOp scan_op, ///< [in] Binary scan operator + T &block_aggregate, ///< [out] block-wide aggregate reduction of input items (exclusive of the \p block_prefix_op value) + BlockPrefixOp &block_prefix_op) ///< [in-out] [warp0 only] Call-back functor for specifying a block-wide prefix to be applied to all inputs. + { + if (ITEMS_PER_THREAD == 1) + { + InclusiveScan(input[0], output[0], scan_op, block_aggregate, block_prefix_op); + } + else + { + // Reduce consecutive thread items in registers + T thread_partial = ThreadReduce(input, scan_op); + + // Exclusive threadblock-scan + ExclusiveScan(thread_partial, thread_partial, scan_op, block_aggregate, block_prefix_op); + + // Inclusive scan in registers with prefix + ThreadScanInclusive(input, output, scan_op, thread_partial); + } + } + + //@} end member group + + +}; + +} // CUB namespace +CUB_NS_POSTFIX // Optional outer namespace(s) + diff --git a/lib/kokkos/TPL/cub/block/block_store.cuh b/lib/kokkos/TPL/cub/block/block_store.cuh new file mode 100755 index 0000000000..fb990de1c7 --- /dev/null +++ b/lib/kokkos/TPL/cub/block/block_store.cuh @@ -0,0 +1,926 @@ +/****************************************************************************** + * Copyright (c) 2011, Duane Merrill. All rights reserved. + * Copyright (c) 2011-2013, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the NVIDIA CORPORATION nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + ******************************************************************************/ + +/** + * \file + * Operations for writing linear segments of data from the CUDA thread block + */ + +#pragma once + +#include + +#include "../util_namespace.cuh" +#include "../util_macro.cuh" +#include "../util_type.cuh" +#include "../util_vector.cuh" +#include "../thread/thread_store.cuh" +#include "block_exchange.cuh" + +/// Optional outer namespace(s) +CUB_NS_PREFIX + +/// CUB namespace +namespace cub { + +/** + * \addtogroup IoModule + * @{ + */ + + +/******************************************************************//** + * \name Blocked I/O + *********************************************************************/ +//@{ + +/** + * \brief Store a blocked arrangement of items across a thread block into a linear segment of items using the specified cache modifier. + * + * \blocked + * + * \tparam MODIFIER cub::PtxStoreModifier cache modifier. + * \tparam T [inferred] The data type to store. + * \tparam ITEMS_PER_THREAD [inferred] The number of consecutive items partitioned onto each thread. + * \tparam OutputIteratorRA [inferred] The random-access iterator type for output (may be a simple pointer type). + */ +template < + PtxStoreModifier MODIFIER, + typename T, + int ITEMS_PER_THREAD, + typename OutputIteratorRA> +__device__ __forceinline__ void StoreBlocked( + int linear_tid, ///< [in] A suitable 1D thread-identifier for the calling thread (e.g., (threadIdx.y * blockDim.x) + linear_tid for 2D thread blocks) + OutputIteratorRA block_itr, ///< [in] The thread block's base output iterator for storing to + T (&items)[ITEMS_PER_THREAD]) ///< [in] Data to store +{ + // Store directly in thread-blocked order + #pragma unroll + for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ITEM++) + { + ThreadStore(block_itr + (linear_tid * ITEMS_PER_THREAD) + ITEM, items[ITEM]); + } +} + + +/** + * \brief Store a blocked arrangement of items across a thread block into a linear segment of items using the specified cache modifier, guarded by range + * + * \blocked + * + * \tparam MODIFIER cub::PtxStoreModifier cache modifier. + * \tparam T [inferred] The data type to store. + * \tparam ITEMS_PER_THREAD [inferred] The number of consecutive items partitioned onto each thread. + * \tparam OutputIteratorRA [inferred] The random-access iterator type for output (may be a simple pointer type). + */ +template < + PtxStoreModifier MODIFIER, + typename T, + int ITEMS_PER_THREAD, + typename OutputIteratorRA> +__device__ __forceinline__ void StoreBlocked( + int linear_tid, ///< [in] A suitable 1D thread-identifier for the calling thread (e.g., (threadIdx.y * blockDim.x) + linear_tid for 2D thread blocks) + OutputIteratorRA block_itr, ///< [in] The thread block's base output iterator for storing to + T (&items)[ITEMS_PER_THREAD], ///< [in] Data to store + int valid_items) ///< [in] Number of valid items to write +{ + // Store directly in thread-blocked order + #pragma unroll + for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ITEM++) + { + if (ITEM + (linear_tid * ITEMS_PER_THREAD) < valid_items) + { + ThreadStore(block_itr + (linear_tid * ITEMS_PER_THREAD) + ITEM, items[ITEM]); + } + } +} + + + +//@} end member group +/******************************************************************//** + * \name Striped I/O + *********************************************************************/ +//@{ + + +/** + * \brief Store a striped arrangement of data across the thread block into a linear segment of items using the specified cache modifier. + * + * \striped + * + * \tparam MODIFIER cub::PtxStoreModifier cache modifier. + * \tparam BLOCK_THREADS The thread block size in threads + * \tparam T [inferred] The data type to store. + * \tparam ITEMS_PER_THREAD [inferred] The number of consecutive items partitioned onto each thread. + * \tparam OutputIteratorRA [inferred] The random-access iterator type for output (may be a simple pointer type). + */ +template < + PtxStoreModifier MODIFIER, + int BLOCK_THREADS, + typename T, + int ITEMS_PER_THREAD, + typename OutputIteratorRA> +__device__ __forceinline__ void StoreStriped( + int linear_tid, ///< [in] A suitable 1D thread-identifier for the calling thread (e.g., (threadIdx.y * blockDim.x) + linear_tid for 2D thread blocks) + OutputIteratorRA block_itr, ///< [in] The thread block's base output iterator for storing to + T (&items)[ITEMS_PER_THREAD]) ///< [in] Data to store +{ + // Store directly in striped order + #pragma unroll + for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ITEM++) + { + ThreadStore(block_itr + (ITEM * BLOCK_THREADS) + linear_tid, items[ITEM]); + } +} + + +/** + * \brief Store a striped arrangement of data across the thread block into a linear segment of items using the specified cache modifier, guarded by range + * + * \striped + * + * \tparam MODIFIER cub::PtxStoreModifier cache modifier. + * \tparam BLOCK_THREADS The thread block size in threads + * \tparam T [inferred] The data type to store. + * \tparam ITEMS_PER_THREAD [inferred] The number of consecutive items partitioned onto each thread. + * \tparam OutputIteratorRA [inferred] The random-access iterator type for output (may be a simple pointer type). + */ +template < + PtxStoreModifier MODIFIER, + int BLOCK_THREADS, + typename T, + int ITEMS_PER_THREAD, + typename OutputIteratorRA> +__device__ __forceinline__ void StoreStriped( + int linear_tid, ///< [in] A suitable 1D thread-identifier for the calling thread (e.g., (threadIdx.y * blockDim.x) + linear_tid for 2D thread blocks) + OutputIteratorRA block_itr, ///< [in] The thread block's base output iterator for storing to + T (&items)[ITEMS_PER_THREAD], ///< [in] Data to store + int valid_items) ///< [in] Number of valid items to write +{ + // Store directly in striped order + #pragma unroll + for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ITEM++) + { + if ((ITEM * BLOCK_THREADS) + linear_tid < valid_items) + { + ThreadStore(block_itr + (ITEM * BLOCK_THREADS) + linear_tid, items[ITEM]); + } + } +} + + + +//@} end member group +/******************************************************************//** + * \name Warp-striped I/O + *********************************************************************/ +//@{ + + +/** + * \brief Store a warp-striped arrangement of data across the thread block into a linear segment of items using the specified cache modifier. + * + * \warpstriped + * + * \par Usage Considerations + * The number of threads in the thread block must be a multiple of the architecture's warp size. + * + * \tparam MODIFIER cub::PtxStoreModifier cache modifier. + * \tparam T [inferred] The data type to store. + * \tparam ITEMS_PER_THREAD [inferred] The number of consecutive items partitioned onto each thread. + * \tparam OutputIteratorRA [inferred] The random-access iterator type for output (may be a simple pointer type). + */ +template < + PtxStoreModifier MODIFIER, + typename T, + int ITEMS_PER_THREAD, + typename OutputIteratorRA> +__device__ __forceinline__ void StoreWarpStriped( + int linear_tid, ///< [in] A suitable 1D thread-identifier for the calling thread (e.g., (threadIdx.y * blockDim.x) + linear_tid for 2D thread blocks) + OutputIteratorRA block_itr, ///< [in] The thread block's base output iterator for storing to + T (&items)[ITEMS_PER_THREAD]) ///< [out] Data to load +{ + int tid = linear_tid & (PtxArchProps::WARP_THREADS - 1); + int wid = linear_tid >> PtxArchProps::LOG_WARP_THREADS; + int warp_offset = wid * PtxArchProps::WARP_THREADS * ITEMS_PER_THREAD; + + // Store directly in warp-striped order + #pragma unroll + for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ITEM++) + { + ThreadStore(block_itr + warp_offset + tid + (ITEM * PtxArchProps::WARP_THREADS), items[ITEM]); + } +} + + +/** + * \brief Store a warp-striped arrangement of data across the thread block into a linear segment of items using the specified cache modifier, guarded by range + * + * \warpstriped + * + * \par Usage Considerations + * The number of threads in the thread block must be a multiple of the architecture's warp size. + * + * \tparam MODIFIER cub::PtxStoreModifier cache modifier. + * \tparam T [inferred] The data type to store. + * \tparam ITEMS_PER_THREAD [inferred] The number of consecutive items partitioned onto each thread. + * \tparam OutputIteratorRA [inferred] The random-access iterator type for output (may be a simple pointer type). + */ +template < + PtxStoreModifier MODIFIER, + typename T, + int ITEMS_PER_THREAD, + typename OutputIteratorRA> +__device__ __forceinline__ void StoreWarpStriped( + int linear_tid, ///< [in] A suitable 1D thread-identifier for the calling thread (e.g., (threadIdx.y * blockDim.x) + linear_tid for 2D thread blocks) + OutputIteratorRA block_itr, ///< [in] The thread block's base output iterator for storing to + T (&items)[ITEMS_PER_THREAD], ///< [in] Data to store + int valid_items) ///< [in] Number of valid items to write +{ + int tid = linear_tid & (PtxArchProps::WARP_THREADS - 1); + int wid = linear_tid >> PtxArchProps::LOG_WARP_THREADS; + int warp_offset = wid * PtxArchProps::WARP_THREADS * ITEMS_PER_THREAD; + + // Store directly in warp-striped order + #pragma unroll + for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ITEM++) + { + if (warp_offset + tid + (ITEM * PtxArchProps::WARP_THREADS) < valid_items) + { + ThreadStore(block_itr + warp_offset + tid + (ITEM * PtxArchProps::WARP_THREADS), items[ITEM]); + } + } +} + + + +//@} end member group +/******************************************************************//** + * \name Blocked, vectorized I/O + *********************************************************************/ +//@{ + +/** + * \brief Store a blocked arrangement of items across a thread block into a linear segment of items using the specified cache modifier. + * + * \blocked + * + * The output offset (\p block_ptr + \p block_offset) must be quad-item aligned, + * which is the default starting offset returned by \p cudaMalloc() + * + * \par + * The following conditions will prevent vectorization and storing will fall back to cub::BLOCK_STORE_DIRECT: + * - \p ITEMS_PER_THREAD is odd + * - The data type \p T is not a built-in primitive or CUDA vector type (e.g., \p short, \p int2, \p double, \p float2, etc.) + * + * \tparam MODIFIER cub::PtxStoreModifier cache modifier. + * \tparam T [inferred] The data type to store. + * \tparam ITEMS_PER_THREAD [inferred] The number of consecutive items partitioned onto each thread. + * + */ +template < + PtxStoreModifier MODIFIER, + typename T, + int ITEMS_PER_THREAD> +__device__ __forceinline__ void StoreBlockedVectorized( + int linear_tid, ///< [in] A suitable 1D thread-identifier for the calling thread (e.g., (threadIdx.y * blockDim.x) + linear_tid for 2D thread blocks) + T *block_ptr, ///< [in] Input pointer for storing from + T (&items)[ITEMS_PER_THREAD]) ///< [in] Data to store +{ + enum + { + // Maximum CUDA vector size is 4 elements + MAX_VEC_SIZE = CUB_MIN(4, ITEMS_PER_THREAD), + + // Vector size must be a power of two and an even divisor of the items per thread + VEC_SIZE = ((((MAX_VEC_SIZE - 1) & MAX_VEC_SIZE) == 0) && ((ITEMS_PER_THREAD % MAX_VEC_SIZE) == 0)) ? + MAX_VEC_SIZE : + 1, + + VECTORS_PER_THREAD = ITEMS_PER_THREAD / VEC_SIZE, + }; + + // Vector type + typedef typename VectorHelper::Type Vector; + + // Alias global pointer + Vector *block_ptr_vectors = reinterpret_cast(block_ptr); + + // Alias pointers (use "raw" array here which should get optimized away to prevent conservative PTXAS lmem spilling) + Vector raw_vector[VECTORS_PER_THREAD]; + T *raw_items = reinterpret_cast(raw_vector); + + // Copy + #pragma unroll + for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ITEM++) + { + raw_items[ITEM] = items[ITEM]; + } + + // Direct-store using vector types + StoreBlocked(linear_tid, block_ptr_vectors, raw_vector); +} + + +//@} end member group + + +/** @} */ // end group IoModule + + +//----------------------------------------------------------------------------- +// Generic BlockStore abstraction +//----------------------------------------------------------------------------- + +/** + * \brief cub::BlockStoreAlgorithm enumerates alternative algorithms for cub::BlockStore to write a blocked arrangement of items across a CUDA thread block to a linear segment of memory. + */ +enum BlockStoreAlgorithm +{ + /** + * \par Overview + * + * A [blocked arrangement](index.html#sec5sec4) of data is written + * directly to memory. The thread block writes items in a parallel "raking" fashion: + * threadi writes the ith segment of consecutive elements. + * + * \par Performance Considerations + * - The utilization of memory transactions (coalescing) decreases as the + * access stride between threads increases (i.e., the number items per thread). + */ + BLOCK_STORE_DIRECT, + + /** + * \par Overview + * + * A [blocked arrangement](index.html#sec5sec4) of data is written directly + * to memory using CUDA's built-in vectorized stores as a coalescing optimization. + * The thread block writes items in a parallel "raking" fashion: threadi uses vector stores to + * write the ith segment of consecutive elements. + * + * For example, st.global.v4.s32 instructions will be generated when \p T = \p int and \p ITEMS_PER_THREAD > 4. + * + * \par Performance Considerations + * - The utilization of memory transactions (coalescing) remains high until the the + * access stride between threads (i.e., the number items per thread) exceeds the + * maximum vector store width (typically 4 items or 64B, whichever is lower). + * - The following conditions will prevent vectorization and writing will fall back to cub::BLOCK_STORE_DIRECT: + * - \p ITEMS_PER_THREAD is odd + * - The \p OutputIteratorRA is not a simple pointer type + * - The block output offset is not quadword-aligned + * - The data type \p T is not a built-in primitive or CUDA vector type (e.g., \p short, \p int2, \p double, \p float2, etc.) + */ + BLOCK_STORE_VECTORIZE, + + /** + * \par Overview + * A [blocked arrangement](index.html#sec5sec4) is locally + * transposed into a [striped arrangement](index.html#sec5sec4) + * which is then written to memory. More specifically, cub::BlockExchange + * used to locally reorder the items into a + * [striped arrangement](index.html#sec5sec4), after which the + * thread block writes items in a parallel "strip-mining" fashion: consecutive + * items owned by threadi are written to memory with + * stride \p BLOCK_THREADS between them. + * + * \par Performance Considerations + * - The utilization of memory transactions (coalescing) remains high regardless + * of items written per thread. + * - The local reordering incurs slightly longer latencies and throughput than the + * direct cub::BLOCK_STORE_DIRECT and cub::BLOCK_STORE_VECTORIZE alternatives. + */ + BLOCK_STORE_TRANSPOSE, + + /** + * \par Overview + * A [blocked arrangement](index.html#sec5sec4) is locally + * transposed into a [warp-striped arrangement](index.html#sec5sec4) + * which is then written to memory. More specifically, cub::BlockExchange used + * to locally reorder the items into a + * [warp-striped arrangement](index.html#sec5sec4), after which + * each warp writes its own contiguous segment in a parallel "strip-mining" fashion: + * consecutive items owned by lanei are written to memory + * with stride \p WARP_THREADS between them. + * + * \par Performance Considerations + * - The utilization of memory transactions (coalescing) remains high regardless + * of items written per thread. + * - The local reordering incurs slightly longer latencies and throughput than the + * direct cub::BLOCK_STORE_DIRECT and cub::BLOCK_STORE_VECTORIZE alternatives. + */ + BLOCK_STORE_WARP_TRANSPOSE, +}; + + + +/** + * \addtogroup BlockModule + * @{ + */ + + +/** + * \brief The BlockStore class provides [collective](index.html#sec0) data movement methods for writing a [blocked arrangement](index.html#sec5sec4) of items partitioned across a CUDA thread block to a linear segment of memory. ![](block_store_logo.png) + * + * \par Overview + * The BlockStore class provides a single data movement abstraction that can be specialized + * to implement different cub::BlockStoreAlgorithm strategies. This facilitates different + * performance policies for different architectures, data types, granularity sizes, etc. + * + * \par Optionally, BlockStore can be specialized by different data movement strategies: + * -# cub::BLOCK_STORE_DIRECT. A [blocked arrangement](index.html#sec5sec4) of data is written + * directly to memory. [More...](\ref cub::BlockStoreAlgorithm) + * -# cub::BLOCK_STORE_VECTORIZE. A [blocked arrangement](index.html#sec5sec4) + * of data is written directly to memory using CUDA's built-in vectorized stores as a + * coalescing optimization. [More...](\ref cub::BlockStoreAlgorithm) + * -# cub::BLOCK_STORE_TRANSPOSE. A [blocked arrangement](index.html#sec5sec4) + * is locally transposed into a [striped arrangement](index.html#sec5sec4) which is + * then written to memory. [More...](\ref cub::BlockStoreAlgorithm) + * -# cub::BLOCK_STORE_WARP_TRANSPOSE. A [blocked arrangement](index.html#sec5sec4) + * is locally transposed into a [warp-striped arrangement](index.html#sec5sec4) which is + * then written to memory. [More...](\ref cub::BlockStoreAlgorithm) + * + * \tparam OutputIteratorRA The input iterator type (may be a simple pointer type). + * \tparam BLOCK_THREADS The thread block size in threads. + * \tparam ITEMS_PER_THREAD The number of consecutive items partitioned onto each thread. + * \tparam ALGORITHM [optional] cub::BlockStoreAlgorithm tuning policy enumeration. default: cub::BLOCK_STORE_DIRECT. + * \tparam MODIFIER [optional] cub::PtxStoreModifier cache modifier. default: cub::STORE_DEFAULT. + * \tparam WARP_TIME_SLICING [optional] For transposition-based cub::BlockStoreAlgorithm parameterizations that utilize shared memory: When \p true, only use enough shared memory for a single warp's worth of data, time-slicing the block-wide exchange over multiple synchronized rounds (default: false) + * + * \par A Simple Example + * \blockcollective{BlockStore} + * \par + * The code snippet below illustrates the storing of a "blocked" arrangement + * of 512 integers across 128 threads (where each thread owns 4 consecutive items) + * into a linear segment of memory. The store is specialized for \p BLOCK_STORE_WARP_TRANSPOSE, + * meaning items are locally reordered among threads so that memory references will be + * efficiently coalesced using a warp-striped access pattern. + * \par + * \code + * #include + * + * __global__ void ExampleKernel(int *d_data, ...) + * { + * // Specialize BlockStore for 128 threads owning 4 integer items each + * typedef cub::BlockStore BlockStore; + * + * // Allocate shared memory for BlockStore + * __shared__ typename BlockStore::TempStorage temp_storage; + * + * // Obtain a segment of consecutive items that are blocked across threads + * int thread_data[4]; + * ... + * + * // Store items to linear memory + * int thread_data[4]; + * BlockStore(temp_storage).Store(d_data, thread_data); + * + * \endcode + * \par + * Suppose the set of \p thread_data across the block of threads is + * { [0,1,2,3], [4,5,6,7], ..., [508,509,510,511] }. + * The output \p d_data will be 0, 1, 2, 3, 4, 5, .... + * + */ +template < + typename OutputIteratorRA, + int BLOCK_THREADS, + int ITEMS_PER_THREAD, + BlockStoreAlgorithm ALGORITHM = BLOCK_STORE_DIRECT, + PtxStoreModifier MODIFIER = STORE_DEFAULT, + bool WARP_TIME_SLICING = false> +class BlockStore +{ +private: + /****************************************************************************** + * Constants and typed definitions + ******************************************************************************/ + + // Data type of input iterator + typedef typename std::iterator_traits::value_type T; + + + /****************************************************************************** + * Algorithmic variants + ******************************************************************************/ + + /// Store helper + template + struct StoreInternal; + + + /** + * BLOCK_STORE_DIRECT specialization of store helper + */ + template + struct StoreInternal + { + /// Shared memory storage layout type + typedef NullType TempStorage; + + /// Linear thread-id + int linear_tid; + + /// Constructor + __device__ __forceinline__ StoreInternal( + TempStorage &temp_storage, + int linear_tid) + : + linear_tid(linear_tid) + {} + + /// Store items into a linear segment of memory + __device__ __forceinline__ void Store( + OutputIteratorRA block_itr, ///< [in] The thread block's base output iterator for storing to + T (&items)[ITEMS_PER_THREAD]) ///< [in] Data to store + { + StoreBlocked(linear_tid, block_itr, items); + } + + /// Store items into a linear segment of memory, guarded by range + __device__ __forceinline__ void Store( + OutputIteratorRA block_itr, ///< [in] The thread block's base output iterator for storing to + T (&items)[ITEMS_PER_THREAD], ///< [in] Data to store + int valid_items) ///< [in] Number of valid items to write + { + StoreBlocked(linear_tid, block_itr, items, valid_items); + } + }; + + + /** + * BLOCK_STORE_VECTORIZE specialization of store helper + */ + template + struct StoreInternal + { + /// Shared memory storage layout type + typedef NullType TempStorage; + + /// Linear thread-id + int linear_tid; + + /// Constructor + __device__ __forceinline__ StoreInternal( + TempStorage &temp_storage, + int linear_tid) + : + linear_tid(linear_tid) + {} + + /// Store items into a linear segment of memory, specialized for native pointer types (attempts vectorization) + __device__ __forceinline__ void Store( + T *block_ptr, ///< [in] The thread block's base output iterator for storing to + T (&items)[ITEMS_PER_THREAD]) ///< [in] Data to store + { + StoreBlockedVectorized(linear_tid, block_ptr, items); + } + + /// Store items into a linear segment of memory, specialized for opaque input iterators (skips vectorization) + template + __device__ __forceinline__ void Store( + _OutputIteratorRA block_itr, ///< [in] The thread block's base output iterator for storing to + T (&items)[ITEMS_PER_THREAD]) ///< [in] Data to store + { + StoreBlocked(linear_tid, block_itr, items); + } + + /// Store items into a linear segment of memory, guarded by range + __device__ __forceinline__ void Store( + OutputIteratorRA block_itr, ///< [in] The thread block's base output iterator for storing to + T (&items)[ITEMS_PER_THREAD], ///< [in] Data to store + int valid_items) ///< [in] Number of valid items to write + { + StoreBlocked(linear_tid, block_itr, items, valid_items); + } + }; + + + /** + * BLOCK_STORE_TRANSPOSE specialization of store helper + */ + template + struct StoreInternal + { + // BlockExchange utility type for keys + typedef BlockExchange BlockExchange; + + /// Shared memory storage layout type + typedef typename BlockExchange::TempStorage _TempStorage; + + /// Alias wrapper allowing storage to be unioned + struct TempStorage : Uninitialized<_TempStorage> {}; + + /// Thread reference to shared storage + _TempStorage &temp_storage; + + /// Linear thread-id + int linear_tid; + + /// Constructor + __device__ __forceinline__ StoreInternal( + TempStorage &temp_storage, + int linear_tid) + : + temp_storage(temp_storage.Alias()), + linear_tid(linear_tid) + {} + + /// Store items into a linear segment of memory + __device__ __forceinline__ void Store( + OutputIteratorRA block_itr, ///< [in] The thread block's base output iterator for storing to + T (&items)[ITEMS_PER_THREAD]) ///< [in] Data to store + { + BlockExchange(temp_storage).BlockedToStriped(items); + StoreStriped(linear_tid, block_itr, items); + } + + /// Store items into a linear segment of memory, guarded by range + __device__ __forceinline__ void Store( + OutputIteratorRA block_itr, ///< [in] The thread block's base output iterator for storing to + T (&items)[ITEMS_PER_THREAD], ///< [in] Data to store + int valid_items) ///< [in] Number of valid items to write + { + BlockExchange(temp_storage).BlockedToStriped(items); + StoreStriped(linear_tid, block_itr, items, valid_items); + } + }; + + + /** + * BLOCK_STORE_WARP_TRANSPOSE specialization of store helper + */ + template + struct StoreInternal + { + enum + { + WARP_THREADS = PtxArchProps::WARP_THREADS + }; + + // Assert BLOCK_THREADS must be a multiple of WARP_THREADS + CUB_STATIC_ASSERT((BLOCK_THREADS % WARP_THREADS == 0), "BLOCK_THREADS must be a multiple of WARP_THREADS"); + + // BlockExchange utility type for keys + typedef BlockExchange BlockExchange; + + /// Shared memory storage layout type + typedef typename BlockExchange::TempStorage _TempStorage; + + /// Alias wrapper allowing storage to be unioned + struct TempStorage : Uninitialized<_TempStorage> {}; + + /// Thread reference to shared storage + _TempStorage &temp_storage; + + /// Linear thread-id + int linear_tid; + + /// Constructor + __device__ __forceinline__ StoreInternal( + TempStorage &temp_storage, + int linear_tid) + : + temp_storage(temp_storage.Alias()), + linear_tid(linear_tid) + {} + + /// Store items into a linear segment of memory + __device__ __forceinline__ void Store( + OutputIteratorRA block_itr, ///< [in] The thread block's base output iterator for storing to + T (&items)[ITEMS_PER_THREAD]) ///< [in] Data to store + { + BlockExchange(temp_storage).BlockedToWarpStriped(items); + StoreWarpStriped(linear_tid, block_itr, items); + } + + /// Store items into a linear segment of memory, guarded by range + __device__ __forceinline__ void Store( + OutputIteratorRA block_itr, ///< [in] The thread block's base output iterator for storing to + T (&items)[ITEMS_PER_THREAD], ///< [in] Data to store + int valid_items) ///< [in] Number of valid items to write + { + BlockExchange(temp_storage).BlockedToWarpStriped(items); + StoreWarpStriped(linear_tid, block_itr, items, valid_items); + } + }; + + /****************************************************************************** + * Type definitions + ******************************************************************************/ + + /// Internal load implementation to use + typedef StoreInternal InternalStore; + + + /// Shared memory storage layout type + typedef typename InternalStore::TempStorage _TempStorage; + + + /****************************************************************************** + * Utility methods + ******************************************************************************/ + + /// Internal storage allocator + __device__ __forceinline__ _TempStorage& PrivateStorage() + { + __shared__ _TempStorage private_storage; + return private_storage; + } + + + /****************************************************************************** + * Thread fields + ******************************************************************************/ + + /// Thread reference to shared storage + _TempStorage &temp_storage; + + /// Linear thread-id + int linear_tid; + +public: + + + /// \smemstorage{BlockStore} + struct TempStorage : Uninitialized<_TempStorage> {}; + + + /******************************************************************//** + * \name Collective constructors + *********************************************************************/ + //@{ + + /** + * \brief Collective constructor for 1D thread blocks using a private static allocation of shared memory as temporary storage. Threads are identified using threadIdx.x. + */ + __device__ __forceinline__ BlockStore() + : + temp_storage(PrivateStorage()), + linear_tid(threadIdx.x) + {} + + + /** + * \brief Collective constructor for 1D thread blocks using the specified memory allocation as temporary storage. Threads are identified using threadIdx.x. + */ + __device__ __forceinline__ BlockStore( + TempStorage &temp_storage) ///< [in] Reference to memory allocation having layout type TempStorage + : + temp_storage(temp_storage.Alias()), + linear_tid(threadIdx.x) + {} + + + /** + * \brief Collective constructor using a private static allocation of shared memory as temporary storage. Each thread is identified using the supplied linear thread identifier + */ + __device__ __forceinline__ BlockStore( + int linear_tid) ///< [in] A suitable 1D thread-identifier for the calling thread (e.g., (threadIdx.y * blockDim.x) + linear_tid for 2D thread blocks) + : + temp_storage(PrivateStorage()), + linear_tid(linear_tid) + {} + + + /** + * \brief Collective constructor using the specified memory allocation as temporary storage. Each thread is identified using the supplied linear thread identifier. + */ + __device__ __forceinline__ BlockStore( + TempStorage &temp_storage, ///< [in] Reference to memory allocation having layout type TempStorage + int linear_tid) ///< [in] [optional] A suitable 1D thread-identifier for the calling thread (e.g., (threadIdx.y * blockDim.x) + linear_tid for 2D thread blocks) + : + temp_storage(temp_storage.Alias()), + linear_tid(linear_tid) + {} + + + //@} end member group + /******************************************************************//** + * \name Data movement + *********************************************************************/ + //@{ + + + /** + * \brief Store items into a linear segment of memory. + * + * \blocked + * + * The code snippet below illustrates the storing of a "blocked" arrangement + * of 512 integers across 128 threads (where each thread owns 4 consecutive items) + * into a linear segment of memory. The store is specialized for \p BLOCK_STORE_WARP_TRANSPOSE, + * meaning items are locally reordered among threads so that memory references will be + * efficiently coalesced using a warp-striped access pattern. + * \par + * \code + * #include + * + * __global__ void ExampleKernel(int *d_data, ...) + * { + * // Specialize BlockStore for 128 threads owning 4 integer items each + * typedef cub::BlockStore BlockStore; + * + * // Allocate shared memory for BlockStore + * __shared__ typename BlockStore::TempStorage temp_storage; + * + * // Obtain a segment of consecutive items that are blocked across threads + * int thread_data[4]; + * ... + * + * // Store items to linear memory + * int thread_data[4]; + * BlockStore(temp_storage).Store(d_data, thread_data); + * + * \endcode + * \par + * Suppose the set of \p thread_data across the block of threads is + * { [0,1,2,3], [4,5,6,7], ..., [508,509,510,511] }. + * The output \p d_data will be 0, 1, 2, 3, 4, 5, .... + * + */ + __device__ __forceinline__ void Store( + OutputIteratorRA block_itr, ///< [in] The thread block's base output iterator for storing to + T (&items)[ITEMS_PER_THREAD]) ///< [in] Data to store + { + InternalStore(temp_storage, linear_tid).Store(block_itr, items); + } + + /** + * \brief Store items into a linear segment of memory, guarded by range. + * + * \blocked + * + * The code snippet below illustrates the guarded storing of a "blocked" arrangement + * of 512 integers across 128 threads (where each thread owns 4 consecutive items) + * into a linear segment of memory. The store is specialized for \p BLOCK_STORE_WARP_TRANSPOSE, + * meaning items are locally reordered among threads so that memory references will be + * efficiently coalesced using a warp-striped access pattern. + * \par + * \code + * #include + * + * __global__ void ExampleKernel(int *d_data, int valid_items, ...) + * { + * // Specialize BlockStore for 128 threads owning 4 integer items each + * typedef cub::BlockStore BlockStore; + * + * // Allocate shared memory for BlockStore + * __shared__ typename BlockStore::TempStorage temp_storage; + * + * // Obtain a segment of consecutive items that are blocked across threads + * int thread_data[4]; + * ... + * + * // Store items to linear memory + * int thread_data[4]; + * BlockStore(temp_storage).Store(d_data, thread_data, valid_items); + * + * \endcode + * \par + * Suppose the set of \p thread_data across the block of threads is + * { [0,1,2,3], [4,5,6,7], ..., [508,509,510,511] } and \p valid_items is \p 5. + * The output \p d_data will be 0, 1, 2, 3, 4, ?, ?, ?, ..., with + * only the first two threads being unmasked to store portions of valid data. + * + */ + __device__ __forceinline__ void Store( + OutputIteratorRA block_itr, ///< [in] The thread block's base output iterator for storing to + T (&items)[ITEMS_PER_THREAD], ///< [in] Data to store + int valid_items) ///< [in] Number of valid items to write + { + InternalStore(temp_storage, linear_tid).Store(block_itr, items, valid_items); + } +}; + +/** @} */ // end group BlockModule + +} // CUB namespace +CUB_NS_POSTFIX // Optional outer namespace(s) + diff --git a/lib/kokkos/TPL/cub/block/specializations/block_histogram_atomic.cuh b/lib/kokkos/TPL/cub/block/specializations/block_histogram_atomic.cuh new file mode 100755 index 0000000000..ecc980098c --- /dev/null +++ b/lib/kokkos/TPL/cub/block/specializations/block_histogram_atomic.cuh @@ -0,0 +1,85 @@ +/****************************************************************************** + * Copyright (c) 2011, Duane Merrill. All rights reserved. + * Copyright (c) 2011-2013, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the NVIDIA CORPORATION nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + ******************************************************************************/ + +/** + * \file + * The cub::BlockHistogramAtomic class provides atomic-based methods for constructing block-wide histograms from data samples partitioned across a CUDA thread block. + */ + +#pragma once + +#include "../../util_namespace.cuh" + +/// Optional outer namespace(s) +CUB_NS_PREFIX + +/// CUB namespace +namespace cub { + + +/** + * \brief The BlockHistogramAtomic class provides atomic-based methods for constructing block-wide histograms from data samples partitioned across a CUDA thread block. + */ +template < + typename T, + int BLOCK_THREADS, + int ITEMS_PER_THREAD, + int BINS> +struct BlockHistogramAtomic +{ + /// Shared memory storage layout type + struct TempStorage {}; + + + /// Constructor + __device__ __forceinline__ BlockHistogramAtomic( + TempStorage &temp_storage, + int linear_tid) + {} + + + /// Composite data onto an existing histogram + template < + typename HistoCounter> + __device__ __forceinline__ void Composite( + T (&items)[ITEMS_PER_THREAD], ///< [in] Calling thread's input values to histogram + HistoCounter histogram[BINS]) ///< [out] Reference to shared/global memory histogram + { + // Update histogram + #pragma unroll + for (int i = 0; i < ITEMS_PER_THREAD; ++i) + { + atomicAdd(histogram + items[i], 1); + } + } + +}; + +} // CUB namespace +CUB_NS_POSTFIX // Optional outer namespace(s) + diff --git a/lib/kokkos/TPL/cub/block/specializations/block_histogram_sort.cuh b/lib/kokkos/TPL/cub/block/specializations/block_histogram_sort.cuh new file mode 100755 index 0000000000..e81edec6c3 --- /dev/null +++ b/lib/kokkos/TPL/cub/block/specializations/block_histogram_sort.cuh @@ -0,0 +1,197 @@ +/****************************************************************************** + * Copyright (c) 2011, Duane Merrill. All rights reserved. + * Copyright (c) 2011-2013, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the NVIDIA CORPORATION nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + ******************************************************************************/ + +/** + * \file + * The cub::BlockHistogramSort class provides sorting-based methods for constructing block-wide histograms from data samples partitioned across a CUDA thread block. + */ + +#pragma once + +#include "../../block/block_radix_sort.cuh" +#include "../../block/block_discontinuity.cuh" +#include "../../util_namespace.cuh" + +/// Optional outer namespace(s) +CUB_NS_PREFIX + +/// CUB namespace +namespace cub { + + + +/** + * \brief The BlockHistogramSort class provides sorting-based methods for constructing block-wide histograms from data samples partitioned across a CUDA thread block. + */ +template < + typename T, + int BLOCK_THREADS, + int ITEMS_PER_THREAD, + int BINS> +struct BlockHistogramSort +{ + // Parameterize BlockRadixSort type for our thread block + typedef BlockRadixSort BlockRadixSortT; + + // Parameterize BlockDiscontinuity type for our thread block + typedef BlockDiscontinuity BlockDiscontinuityT; + + // Shared memory + union _TempStorage + { + // Storage for sorting bin values + typename BlockRadixSortT::TempStorage sort; + + struct + { + // Storage for detecting discontinuities in the tile of sorted bin values + typename BlockDiscontinuityT::TempStorage flag; + + // Storage for noting begin/end offsets of bin runs in the tile of sorted bin values + unsigned int run_begin[BINS]; + unsigned int run_end[BINS]; + }; + }; + + + /// Alias wrapper allowing storage to be unioned + struct TempStorage : Uninitialized<_TempStorage> {}; + + + // Thread fields + _TempStorage &temp_storage; + int linear_tid; + + + /// Constructor + __device__ __forceinline__ BlockHistogramSort( + TempStorage &temp_storage, + int linear_tid) + : + temp_storage(temp_storage.Alias()), + linear_tid(linear_tid) + {} + + + // Discontinuity functor + struct DiscontinuityOp + { + // Reference to temp_storage + _TempStorage &temp_storage; + + // Constructor + __device__ __forceinline__ DiscontinuityOp(_TempStorage &temp_storage) : + temp_storage(temp_storage) + {} + + // Discontinuity predicate + __device__ __forceinline__ bool operator()(const T &a, const T &b, unsigned int b_index) + { + if (a != b) + { + // Note the begin/end offsets in shared storage + temp_storage.run_begin[b] = b_index; + temp_storage.run_end[a] = b_index; + + return true; + } + else + { + return false; + } + } + }; + + + // Composite data onto an existing histogram + template < + typename HistoCounter> + __device__ __forceinline__ void Composite( + T (&items)[ITEMS_PER_THREAD], ///< [in] Calling thread's input values to histogram + HistoCounter histogram[BINS]) ///< [out] Reference to shared/global memory histogram + { + enum { TILE_SIZE = BLOCK_THREADS * ITEMS_PER_THREAD }; + + // Sort bytes in blocked arrangement + BlockRadixSortT(temp_storage.sort, linear_tid).Sort(items); + + __syncthreads(); + + // Initialize the shared memory's run_begin and run_end for each bin + int histo_offset = 0; + + #pragma unroll + for(; histo_offset + BLOCK_THREADS <= BINS; histo_offset += BLOCK_THREADS) + { + temp_storage.run_begin[histo_offset + linear_tid] = TILE_SIZE; + temp_storage.run_end[histo_offset + linear_tid] = TILE_SIZE; + } + // Finish up with guarded initialization if necessary + if ((BINS % BLOCK_THREADS != 0) && (histo_offset + linear_tid < BINS)) + { + temp_storage.run_begin[histo_offset + linear_tid] = TILE_SIZE; + temp_storage.run_end[histo_offset + linear_tid] = TILE_SIZE; + } + + __syncthreads(); + + int flags[ITEMS_PER_THREAD]; // unused + + // Compute head flags to demarcate contiguous runs of the same bin in the sorted tile + DiscontinuityOp flag_op(temp_storage); + BlockDiscontinuityT(temp_storage.flag, linear_tid).FlagHeads(flags, items, flag_op); + + // Update begin for first item + if (linear_tid == 0) temp_storage.run_begin[items[0]] = 0; + + __syncthreads(); + + // Composite into histogram + histo_offset = 0; + + #pragma unroll + for(; histo_offset + BLOCK_THREADS <= BINS; histo_offset += BLOCK_THREADS) + { + int thread_offset = histo_offset + linear_tid; + HistoCounter count = temp_storage.run_end[thread_offset] - temp_storage.run_begin[thread_offset]; + histogram[thread_offset] += count; + } + // Finish up with guarded composition if necessary + if ((BINS % BLOCK_THREADS != 0) && (histo_offset + linear_tid < BINS)) + { + int thread_offset = histo_offset + linear_tid; + HistoCounter count = temp_storage.run_end[thread_offset] - temp_storage.run_begin[thread_offset]; + histogram[thread_offset] += count; + } + } + +}; + +} // CUB namespace +CUB_NS_POSTFIX // Optional outer namespace(s) + diff --git a/lib/kokkos/TPL/cub/block/specializations/block_reduce_raking.cuh b/lib/kokkos/TPL/cub/block/specializations/block_reduce_raking.cuh new file mode 100755 index 0000000000..434d25a872 --- /dev/null +++ b/lib/kokkos/TPL/cub/block/specializations/block_reduce_raking.cuh @@ -0,0 +1,214 @@ +/****************************************************************************** + * Copyright (c) 2011, Duane Merrill. All rights reserved. + * Copyright (c) 2011-2013, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the NVIDIA CORPORATION nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + ******************************************************************************/ + +/** + * \file + * cub::BlockReduceRaking provides raking-based methods of parallel reduction across a CUDA threadblock + */ + +#pragma once + +#include "../../block/block_raking_layout.cuh" +#include "../../warp/warp_reduce.cuh" +#include "../../thread/thread_reduce.cuh" +#include "../../util_namespace.cuh" + +/// Optional outer namespace(s) +CUB_NS_PREFIX + +/// CUB namespace +namespace cub { + + +/** + * \brief BlockReduceRaking provides raking-based methods of parallel reduction across a CUDA threadblock + */ +template < + typename T, ///< Data type being reduced + int BLOCK_THREADS> ///< The thread block size in threads +struct BlockReduceRaking +{ + /// Layout type for padded threadblock raking grid + typedef BlockRakingLayout BlockRakingLayout; + + /// WarpReduce utility type + typedef typename WarpReduce::InternalWarpReduce WarpReduce; + + /// Constants + enum + { + /// Number of raking threads + RAKING_THREADS = BlockRakingLayout::RAKING_THREADS, + + /// Number of raking elements per warp synchronous raking thread + SEGMENT_LENGTH = BlockRakingLayout::SEGMENT_LENGTH, + + /// Cooperative work can be entirely warp synchronous + WARP_SYNCHRONOUS = (RAKING_THREADS == BLOCK_THREADS), + + /// Whether or not warp-synchronous reduction should be unguarded (i.e., the warp-reduction elements is a power of two + WARP_SYNCHRONOUS_UNGUARDED = ((RAKING_THREADS & (RAKING_THREADS - 1)) == 0), + + /// Whether or not accesses into smem are unguarded + RAKING_UNGUARDED = BlockRakingLayout::UNGUARDED, + + }; + + + /// Shared memory storage layout type + struct _TempStorage + { + typename WarpReduce::TempStorage warp_storage; ///< Storage for warp-synchronous reduction + typename BlockRakingLayout::TempStorage raking_grid; ///< Padded threadblock raking grid + }; + + + /// Alias wrapper allowing storage to be unioned + struct TempStorage : Uninitialized<_TempStorage> {}; + + + // Thread fields + _TempStorage &temp_storage; + int linear_tid; + + + /// Constructor + __device__ __forceinline__ BlockReduceRaking( + TempStorage &temp_storage, + int linear_tid) + : + temp_storage(temp_storage.Alias()), + linear_tid(linear_tid) + {} + + + /// Computes a threadblock-wide reduction using addition (+) as the reduction operator. The first num_valid threads each contribute one reduction partial. The return value is only valid for thread0. + template + __device__ __forceinline__ T Sum( + T partial, ///< [in] Calling thread's input partial reductions + int num_valid) ///< [in] Number of valid elements (may be less than BLOCK_THREADS) + { + cub::Sum reduction_op; + + if (WARP_SYNCHRONOUS) + { + // Short-circuit directly to warp synchronous reduction (unguarded if active threads is a power-of-two) + partial = WarpReduce(temp_storage.warp_storage, 0, linear_tid).template Sum( + partial, + num_valid); + } + else + { + // Place partial into shared memory grid. + *BlockRakingLayout::PlacementPtr(temp_storage.raking_grid, linear_tid) = partial; + + __syncthreads(); + + // Reduce parallelism to one warp + if (linear_tid < RAKING_THREADS) + { + // Raking reduction in grid + T *raking_segment = BlockRakingLayout::RakingPtr(temp_storage.raking_grid, linear_tid); + partial = raking_segment[0]; + + #pragma unroll + for (int ITEM = 1; ITEM < SEGMENT_LENGTH; ITEM++) + { + // Update partial if addend is in range + if ((FULL_TILE && RAKING_UNGUARDED) || ((linear_tid * SEGMENT_LENGTH) + ITEM < num_valid)) + { + partial = reduction_op(partial, raking_segment[ITEM]); + } + } + + partial = WarpReduce(temp_storage.warp_storage, 0, linear_tid).template Sum( + partial, + num_valid); + } + } + + return partial; + } + + + /// Computes a threadblock-wide reduction using the specified reduction operator. The first num_valid threads each contribute one reduction partial. The return value is only valid for thread0. + template < + bool FULL_TILE, + typename ReductionOp> + __device__ __forceinline__ T Reduce( + T partial, ///< [in] Calling thread's input partial reductions + int num_valid, ///< [in] Number of valid elements (may be less than BLOCK_THREADS) + ReductionOp reduction_op) ///< [in] Binary reduction operator + { + if (WARP_SYNCHRONOUS) + { + // Short-circuit directly to warp synchronous reduction (unguarded if active threads is a power-of-two) + partial = WarpReduce(temp_storage.warp_storage, 0, linear_tid).template Reduce( + partial, + num_valid, + reduction_op); + } + else + { + // Place partial into shared memory grid. + *BlockRakingLayout::PlacementPtr(temp_storage.raking_grid, linear_tid) = partial; + + __syncthreads(); + + // Reduce parallelism to one warp + if (linear_tid < RAKING_THREADS) + { + // Raking reduction in grid + T *raking_segment = BlockRakingLayout::RakingPtr(temp_storage.raking_grid, linear_tid); + partial = raking_segment[0]; + + #pragma unroll + for (int ITEM = 1; ITEM < SEGMENT_LENGTH; ITEM++) + { + // Update partial if addend is in range + if ((FULL_TILE && RAKING_UNGUARDED) || ((linear_tid * SEGMENT_LENGTH) + ITEM < num_valid)) + { + partial = reduction_op(partial, raking_segment[ITEM]); + } + } + + partial = WarpReduce(temp_storage.warp_storage, 0, linear_tid).template Reduce( + partial, + num_valid, + reduction_op); + } + } + + return partial; + } + +}; + +} // CUB namespace +CUB_NS_POSTFIX // Optional outer namespace(s) + diff --git a/lib/kokkos/TPL/cub/block/specializations/block_reduce_warp_reductions.cuh b/lib/kokkos/TPL/cub/block/specializations/block_reduce_warp_reductions.cuh new file mode 100755 index 0000000000..0e316dd17e --- /dev/null +++ b/lib/kokkos/TPL/cub/block/specializations/block_reduce_warp_reductions.cuh @@ -0,0 +1,198 @@ +/****************************************************************************** + * Copyright (c) 2011, Duane Merrill. All rights reserved. + * Copyright (c) 2011-2013, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the NVIDIA CORPORATION nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + ******************************************************************************/ + +/** + * \file + * cub::BlockReduceWarpReductions provides variants of warp-reduction-based parallel reduction across a CUDA threadblock + */ + +#pragma once + +#include "../../warp/warp_reduce.cuh" +#include "../../util_arch.cuh" +#include "../../util_namespace.cuh" + +/// Optional outer namespace(s) +CUB_NS_PREFIX + +/// CUB namespace +namespace cub { + + +/** + * \brief BlockReduceWarpReductions provides variants of warp-reduction-based parallel reduction across a CUDA threadblock + */ +template < + typename T, ///< Data type being reduced + int BLOCK_THREADS> ///< The thread block size in threads +struct BlockReduceWarpReductions +{ + /// Constants + enum + { + /// Number of active warps + WARPS = (BLOCK_THREADS + PtxArchProps::WARP_THREADS - 1) / PtxArchProps::WARP_THREADS, + + /// The logical warp size for warp reductions + LOGICAL_WARP_SIZE = CUB_MIN(BLOCK_THREADS, PtxArchProps::WARP_THREADS), + + /// Whether or not the logical warp size evenly divides the threadblock size + EVEN_WARP_MULTIPLE = (BLOCK_THREADS % LOGICAL_WARP_SIZE == 0) + }; + + + /// WarpReduce utility type + typedef typename WarpReduce::InternalWarpReduce WarpReduce; + + + /// Shared memory storage layout type + struct _TempStorage + { + typename WarpReduce::TempStorage warp_reduce; ///< Buffer for warp-synchronous scan + T warp_aggregates[WARPS]; ///< Shared totals from each warp-synchronous scan + T block_prefix; ///< Shared prefix for the entire threadblock + }; + + /// Alias wrapper allowing storage to be unioned + struct TempStorage : Uninitialized<_TempStorage> {}; + + + // Thread fields + _TempStorage &temp_storage; + int linear_tid; + int warp_id; + int lane_id; + + + /// Constructor + __device__ __forceinline__ BlockReduceWarpReductions( + TempStorage &temp_storage, + int linear_tid) + : + temp_storage(temp_storage.Alias()), + linear_tid(linear_tid), + warp_id((BLOCK_THREADS <= PtxArchProps::WARP_THREADS) ? + 0 : + linear_tid / PtxArchProps::WARP_THREADS), + lane_id((BLOCK_THREADS <= PtxArchProps::WARP_THREADS) ? + linear_tid : + linear_tid % PtxArchProps::WARP_THREADS) + {} + + + /// Returns block-wide aggregate in thread0. + template < + bool FULL_TILE, + typename ReductionOp> + __device__ __forceinline__ T ApplyWarpAggregates( + ReductionOp reduction_op, ///< [in] Binary scan operator + T warp_aggregate, ///< [in] [lane0s only] Warp-wide aggregate reduction of input items + int num_valid) ///< [in] Number of valid elements (may be less than BLOCK_THREADS) + { + // Share lane aggregates + if (lane_id == 0) + { + temp_storage.warp_aggregates[warp_id] = warp_aggregate; + } + + __syncthreads(); + + // Update total aggregate in warp 0, lane 0 + if (linear_tid == 0) + { + #pragma unroll + for (int SUCCESSOR_WARP = 1; SUCCESSOR_WARP < WARPS; SUCCESSOR_WARP++) + { + if (FULL_TILE || (SUCCESSOR_WARP * LOGICAL_WARP_SIZE < num_valid)) + { + warp_aggregate = reduction_op(warp_aggregate, temp_storage.warp_aggregates[SUCCESSOR_WARP]); + } + } + } + + return warp_aggregate; + } + + + /// Computes a threadblock-wide reduction using addition (+) as the reduction operator. The first num_valid threads each contribute one reduction partial. The return value is only valid for thread0. + template + __device__ __forceinline__ T Sum( + T input, ///< [in] Calling thread's input partial reductions + int num_valid) ///< [in] Number of valid elements (may be less than BLOCK_THREADS) + { + cub::Sum reduction_op; + unsigned int warp_offset = warp_id * LOGICAL_WARP_SIZE; + unsigned int warp_num_valid = (FULL_TILE && EVEN_WARP_MULTIPLE) ? + LOGICAL_WARP_SIZE : + (warp_offset < num_valid) ? + num_valid - warp_offset : + 0; + + // Warp reduction in every warp + T warp_aggregate = WarpReduce(temp_storage.warp_reduce, warp_id, lane_id).template Sum<(FULL_TILE && EVEN_WARP_MULTIPLE), 1>( + input, + warp_num_valid); + + // Update outputs and block_aggregate with warp-wide aggregates from lane-0s + return ApplyWarpAggregates(reduction_op, warp_aggregate, num_valid); + } + + + /// Computes a threadblock-wide reduction using the specified reduction operator. The first num_valid threads each contribute one reduction partial. The return value is only valid for thread0. + template < + bool FULL_TILE, + typename ReductionOp> + __device__ __forceinline__ T Reduce( + T input, ///< [in] Calling thread's input partial reductions + int num_valid, ///< [in] Number of valid elements (may be less than BLOCK_THREADS) + ReductionOp reduction_op) ///< [in] Binary reduction operator + { + unsigned int warp_id = (WARPS == 1) ? 0 : (linear_tid / LOGICAL_WARP_SIZE); + unsigned int warp_offset = warp_id * LOGICAL_WARP_SIZE; + unsigned int warp_num_valid = (FULL_TILE && EVEN_WARP_MULTIPLE) ? + LOGICAL_WARP_SIZE : + (warp_offset < num_valid) ? + num_valid - warp_offset : + 0; + + // Warp reduction in every warp + T warp_aggregate = WarpReduce(temp_storage.warp_reduce, warp_id, lane_id).template Reduce<(FULL_TILE && EVEN_WARP_MULTIPLE), 1>( + input, + warp_num_valid, + reduction_op); + + // Update outputs and block_aggregate with warp-wide aggregates from lane-0s + return ApplyWarpAggregates(reduction_op, warp_aggregate, num_valid); + } + +}; + + +} // CUB namespace +CUB_NS_POSTFIX // Optional outer namespace(s) + diff --git a/lib/kokkos/TPL/cub/block/specializations/block_scan_raking.cuh b/lib/kokkos/TPL/cub/block/specializations/block_scan_raking.cuh new file mode 100755 index 0000000000..75e15d95cf --- /dev/null +++ b/lib/kokkos/TPL/cub/block/specializations/block_scan_raking.cuh @@ -0,0 +1,761 @@ +/****************************************************************************** + * Copyright (c) 2011, Duane Merrill. All rights reserved. + * Copyright (c) 2011-2013, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the NVIDIA CORPORATION nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + ******************************************************************************/ + + +/** + * \file + * cub::BlockScanRaking provides variants of raking-based parallel prefix scan across a CUDA threadblock. + */ + +#pragma once + +#include "../../util_arch.cuh" +#include "../../block/block_raking_layout.cuh" +#include "../../thread/thread_reduce.cuh" +#include "../../thread/thread_scan.cuh" +#include "../../warp/warp_scan.cuh" +#include "../../util_namespace.cuh" + +/// Optional outer namespace(s) +CUB_NS_PREFIX + +/// CUB namespace +namespace cub { + + +/** + * \brief BlockScanRaking provides variants of raking-based parallel prefix scan across a CUDA threadblock. + */ +template < + typename T, ///< Data type being scanned + int BLOCK_THREADS, ///< The thread block size in threads + bool MEMOIZE> ///< Whether or not to buffer outer raking scan partials to incur fewer shared memory reads at the expense of higher register pressure +struct BlockScanRaking +{ + /// Layout type for padded threadblock raking grid + typedef BlockRakingLayout BlockRakingLayout; + + /// Constants + enum + { + /// Number of active warps + WARPS = (BLOCK_THREADS + PtxArchProps::WARP_THREADS - 1) / PtxArchProps::WARP_THREADS, + + /// Number of raking threads + RAKING_THREADS = BlockRakingLayout::RAKING_THREADS, + + /// Number of raking elements per warp synchronous raking thread + SEGMENT_LENGTH = BlockRakingLayout::SEGMENT_LENGTH, + + /// Cooperative work can be entirely warp synchronous + WARP_SYNCHRONOUS = (BLOCK_THREADS == RAKING_THREADS), + }; + + /// WarpScan utility type + typedef WarpScan WarpScan; + + /// Shared memory storage layout type + struct _TempStorage + { + typename WarpScan::TempStorage warp_scan; ///< Buffer for warp-synchronous scan + typename BlockRakingLayout::TempStorage raking_grid; ///< Padded threadblock raking grid + T block_aggregate; ///< Block aggregate + }; + + + /// Alias wrapper allowing storage to be unioned + struct TempStorage : Uninitialized<_TempStorage> {}; + + + // Thread fields + _TempStorage &temp_storage; + int linear_tid; + T cached_segment[SEGMENT_LENGTH]; + + + /// Constructor + __device__ __forceinline__ BlockScanRaking( + TempStorage &temp_storage, + int linear_tid) + : + temp_storage(temp_storage.Alias()), + linear_tid(linear_tid) + {} + + /// Performs upsweep raking reduction, returning the aggregate + template + __device__ __forceinline__ T Upsweep( + ScanOp scan_op) + { + T *smem_raking_ptr = BlockRakingLayout::RakingPtr(temp_storage.raking_grid, linear_tid); + T *raking_ptr; + + if (MEMOIZE) + { + // Copy data into registers + #pragma unroll + for (int i = 0; i < SEGMENT_LENGTH; i++) + { + cached_segment[i] = smem_raking_ptr[i]; + } + raking_ptr = cached_segment; + } + else + { + raking_ptr = smem_raking_ptr; + } + + T raking_partial = raking_ptr[0]; + + #pragma unroll + for (int i = 1; i < SEGMENT_LENGTH; i++) + { + if ((BlockRakingLayout::UNGUARDED) || (((linear_tid * SEGMENT_LENGTH) + i) < BLOCK_THREADS)) + { + raking_partial = scan_op(raking_partial, raking_ptr[i]); + } + } + + return raking_partial; + } + + + /// Performs exclusive downsweep raking scan + template + __device__ __forceinline__ void ExclusiveDownsweep( + ScanOp scan_op, + T raking_partial, + bool apply_prefix = true) + { + T *smem_raking_ptr = BlockRakingLayout::RakingPtr(temp_storage.raking_grid, linear_tid); + + T *raking_ptr = (MEMOIZE) ? + cached_segment : + smem_raking_ptr; + + ThreadScanExclusive(raking_ptr, raking_ptr, scan_op, raking_partial, apply_prefix); + + if (MEMOIZE) + { + // Copy data back to smem + #pragma unroll + for (int i = 0; i < SEGMENT_LENGTH; i++) + { + smem_raking_ptr[i] = cached_segment[i]; + } + } + } + + + /// Performs inclusive downsweep raking scan + template + __device__ __forceinline__ void InclusiveDownsweep( + ScanOp scan_op, + T raking_partial, + bool apply_prefix = true) + { + T *smem_raking_ptr = BlockRakingLayout::RakingPtr(temp_storage.raking_grid, linear_tid); + + T *raking_ptr = (MEMOIZE) ? + cached_segment : + smem_raking_ptr; + + ThreadScanInclusive(raking_ptr, raking_ptr, scan_op, raking_partial, apply_prefix); + + if (MEMOIZE) + { + // Copy data back to smem + #pragma unroll + for (int i = 0; i < SEGMENT_LENGTH; i++) + { + smem_raking_ptr[i] = cached_segment[i]; + } + } + } + + + /// Computes an exclusive threadblock-wide prefix scan using the specified binary \p scan_op functor. Each thread contributes one input element. Also provides every thread with the block-wide \p block_aggregate of all inputs. + template + __device__ __forceinline__ void ExclusiveScan( + T input, ///< [in] Calling thread's input items + T &output, ///< [out] Calling thread's output items (may be aliased to \p input) + const T &identity, ///< [in] Identity value + ScanOp scan_op, ///< [in] Binary scan operator + T &block_aggregate) ///< [out] Threadblock-wide aggregate reduction of input items + { + if (WARP_SYNCHRONOUS) + { + // Short-circuit directly to warp scan + WarpScan(temp_storage.warp_scan, 0, linear_tid).ExclusiveScan( + input, + output, + identity, + scan_op, + block_aggregate); + } + else + { + // Place thread partial into shared memory raking grid + T *placement_ptr = BlockRakingLayout::PlacementPtr(temp_storage.raking_grid, linear_tid); + *placement_ptr = input; + + __syncthreads(); + + // Reduce parallelism down to just raking threads + if (linear_tid < RAKING_THREADS) + { + // Raking upsweep reduction in grid + T raking_partial = Upsweep(scan_op); + + // Exclusive warp synchronous scan + WarpScan(temp_storage.warp_scan, 0, linear_tid).ExclusiveScan( + raking_partial, + raking_partial, + identity, + scan_op, + temp_storage.block_aggregate); + + // Exclusive raking downsweep scan + ExclusiveDownsweep(scan_op, raking_partial); + } + + __syncthreads(); + + // Grab thread prefix from shared memory + output = *placement_ptr; + + // Retrieve block aggregate + block_aggregate = temp_storage.block_aggregate; + } + } + + + /// Computes an exclusive threadblock-wide prefix scan using the specified binary \p scan_op functor. Each thread contributes one input element. the call-back functor \p block_prefix_op is invoked by the first warp in the block, and the value returned by lane0 in that warp is used as the "seed" value that logically prefixes the threadblock's scan inputs. Also provides every thread with the block-wide \p block_aggregate of all inputs. + template < + typename ScanOp, + typename BlockPrefixOp> + __device__ __forceinline__ void ExclusiveScan( + T input, ///< [in] Calling thread's input item + T &output, ///< [out] Calling thread's output item (may be aliased to \p input) + T identity, ///< [in] Identity value + ScanOp scan_op, ///< [in] Binary scan operator + T &block_aggregate, ///< [out] Threadblock-wide aggregate reduction of input items (exclusive of the \p block_prefix_op value) + BlockPrefixOp &block_prefix_op) ///< [in-out] [warp0 only] Call-back functor for specifying a threadblock-wide prefix to be applied to all inputs. + { + if (WARP_SYNCHRONOUS) + { + // Short-circuit directly to warp scan + WarpScan(temp_storage.warp_scan, 0, linear_tid).ExclusiveScan( + input, + output, + identity, + scan_op, + block_aggregate, + block_prefix_op); + } + else + { + // Place thread partial into shared memory raking grid + T *placement_ptr = BlockRakingLayout::PlacementPtr(temp_storage.raking_grid, linear_tid); + *placement_ptr = input; + + __syncthreads(); + + // Reduce parallelism down to just raking threads + if (linear_tid < RAKING_THREADS) + { + // Raking upsweep reduction in grid + T raking_partial = Upsweep(scan_op); + + // Exclusive warp synchronous scan + WarpScan(temp_storage.warp_scan, 0, linear_tid).ExclusiveScan( + raking_partial, + raking_partial, + identity, + scan_op, + temp_storage.block_aggregate, + block_prefix_op); + + // Exclusive raking downsweep scan + ExclusiveDownsweep(scan_op, raking_partial); + } + + __syncthreads(); + + // Grab thread prefix from shared memory + output = *placement_ptr; + + // Retrieve block aggregate + block_aggregate = temp_storage.block_aggregate; + } + } + + + /// Computes an exclusive threadblock-wide prefix scan using the specified binary \p scan_op functor. Each thread contributes one input element. Also provides every thread with the block-wide \p block_aggregate of all inputs. With no identity value, the output computed for thread0 is undefined. + template + __device__ __forceinline__ void ExclusiveScan( + T input, ///< [in] Calling thread's input item + T &output, ///< [out] Calling thread's output item (may be aliased to \p input) + ScanOp scan_op, ///< [in] Binary scan operator + T &block_aggregate) ///< [out] Threadblock-wide aggregate reduction of input items + { + if (WARP_SYNCHRONOUS) + { + // Short-circuit directly to warp scan + WarpScan(temp_storage.warp_scan, 0, linear_tid).ExclusiveScan( + input, + output, + scan_op, + block_aggregate); + } + else + { + // Place thread partial into shared memory raking grid + T *placement_ptr = BlockRakingLayout::PlacementPtr(temp_storage.raking_grid, linear_tid); + *placement_ptr = input; + + __syncthreads(); + + // Reduce parallelism down to just raking threads + if (linear_tid < RAKING_THREADS) + { + // Raking upsweep reduction in grid + T raking_partial = Upsweep(scan_op); + + // Exclusive warp synchronous scan + WarpScan(temp_storage.warp_scan, 0, linear_tid).ExclusiveScan( + raking_partial, + raking_partial, + scan_op, + temp_storage.block_aggregate); + + // Exclusive raking downsweep scan + ExclusiveDownsweep(scan_op, raking_partial, (linear_tid != 0)); + } + + __syncthreads(); + + // Grab thread prefix from shared memory + output = *placement_ptr; + + // Retrieve block aggregate + block_aggregate = temp_storage.block_aggregate; + } + } + + + /// Computes an exclusive threadblock-wide prefix scan using the specified binary \p scan_op functor. Each thread contributes one input element. the call-back functor \p block_prefix_op is invoked by the first warp in the block, and the value returned by lane0 in that warp is used as the "seed" value that logically prefixes the threadblock's scan inputs. Also provides every thread with the block-wide \p block_aggregate of all inputs. + template < + typename ScanOp, + typename BlockPrefixOp> + __device__ __forceinline__ void ExclusiveScan( + T input, ///< [in] Calling thread's input item + T &output, ///< [out] Calling thread's output item (may be aliased to \p input) + ScanOp scan_op, ///< [in] Binary scan operator + T &block_aggregate, ///< [out] Threadblock-wide aggregate reduction of input items (exclusive of the \p block_prefix_op value) + BlockPrefixOp &block_prefix_op) ///< [in-out] [warp0 only] Call-back functor for specifying a threadblock-wide prefix to be applied to all inputs. + { + if (WARP_SYNCHRONOUS) + { + // Short-circuit directly to warp scan + WarpScan(temp_storage.warp_scan, 0, linear_tid).ExclusiveScan( + input, + output, + scan_op, + block_aggregate, + block_prefix_op); + } + else + { + // Place thread partial into shared memory raking grid + T *placement_ptr = BlockRakingLayout::PlacementPtr(temp_storage.raking_grid, linear_tid); + *placement_ptr = input; + + __syncthreads(); + + // Reduce parallelism down to just raking threads + if (linear_tid < RAKING_THREADS) + { + // Raking upsweep reduction in grid + T raking_partial = Upsweep(scan_op); + + // Exclusive warp synchronous scan + WarpScan(temp_storage.warp_scan, 0, linear_tid).ExclusiveScan( + raking_partial, + raking_partial, + scan_op, + temp_storage.block_aggregate, + block_prefix_op); + + // Exclusive raking downsweep scan + ExclusiveDownsweep(scan_op, raking_partial); + } + + __syncthreads(); + + // Grab thread prefix from shared memory + output = *placement_ptr; + + // Retrieve block aggregate + block_aggregate = temp_storage.block_aggregate; + } + } + + + /// Computes an exclusive threadblock-wide prefix scan using addition (+) as the scan operator. Each thread contributes one input element. Also provides every thread with the block-wide \p block_aggregate of all inputs. + __device__ __forceinline__ void ExclusiveSum( + T input, ///< [in] Calling thread's input item + T &output, ///< [out] Calling thread's output item (may be aliased to \p input) + T &block_aggregate) ///< [out] Threadblock-wide aggregate reduction of input items + { + if (WARP_SYNCHRONOUS) + { + // Short-circuit directly to warp scan + WarpScan(temp_storage.warp_scan, 0, linear_tid).ExclusiveSum( + input, + output, + block_aggregate); + } + else + { + // Raking scan + Sum scan_op; + + // Place thread partial into shared memory raking grid + T *placement_ptr = BlockRakingLayout::PlacementPtr(temp_storage.raking_grid, linear_tid); + *placement_ptr = input; + + __syncthreads(); + + // Reduce parallelism down to just raking threads + if (linear_tid < RAKING_THREADS) + { + // Raking upsweep reduction in grid + T raking_partial = Upsweep(scan_op); + + // Exclusive warp synchronous scan + WarpScan(temp_storage.warp_scan, 0, linear_tid).ExclusiveSum( + raking_partial, + raking_partial, + temp_storage.block_aggregate); + + // Exclusive raking downsweep scan + ExclusiveDownsweep(scan_op, raking_partial); + } + + __syncthreads(); + + // Grab thread prefix from shared memory + output = *placement_ptr; + + // Retrieve block aggregate + block_aggregate = temp_storage.block_aggregate; + } + } + + + /// Computes an exclusive threadblock-wide prefix scan using addition (+) as the scan operator. Each thread contributes one input element. Instead of using 0 as the threadblock-wide prefix, the call-back functor \p block_prefix_op is invoked by the first warp in the block, and the value returned by lane0 in that warp is used as the "seed" value that logically prefixes the threadblock's scan inputs. Also provides every thread with the block-wide \p block_aggregate of all inputs. + template + __device__ __forceinline__ void ExclusiveSum( + T input, ///< [in] Calling thread's input item + T &output, ///< [out] Calling thread's output item (may be aliased to \p input) + T &block_aggregate, ///< [out] Threadblock-wide aggregate reduction of input items (exclusive of the \p block_prefix_op value) + BlockPrefixOp &block_prefix_op) ///< [in-out] [warp0 only] Call-back functor for specifying a threadblock-wide prefix to be applied to all inputs. + { + if (WARP_SYNCHRONOUS) + { + // Short-circuit directly to warp scan + WarpScan(temp_storage.warp_scan, 0, linear_tid).ExclusiveSum( + input, + output, + block_aggregate, + block_prefix_op); + } + else + { + // Raking scan + Sum scan_op; + + // Place thread partial into shared memory raking grid + T *placement_ptr = BlockRakingLayout::PlacementPtr(temp_storage.raking_grid, linear_tid); + *placement_ptr = input; + + __syncthreads(); + + // Reduce parallelism down to just raking threads + if (linear_tid < RAKING_THREADS) + { + // Raking upsweep reduction in grid + T raking_partial = Upsweep(scan_op); + + // Exclusive warp synchronous scan + WarpScan(temp_storage.warp_scan, 0, linear_tid).ExclusiveSum( + raking_partial, + raking_partial, + temp_storage.block_aggregate, + block_prefix_op); + + // Exclusive raking downsweep scan + ExclusiveDownsweep(scan_op, raking_partial); + } + + __syncthreads(); + + // Grab thread prefix from shared memory + output = *placement_ptr; + + // Retrieve block aggregate + block_aggregate = temp_storage.block_aggregate; + } + } + + + /// Computes an inclusive threadblock-wide prefix scan using the specified binary \p scan_op functor. Each thread contributes one input element. Also provides every thread with the block-wide \p block_aggregate of all inputs. + template + __device__ __forceinline__ void InclusiveScan( + T input, ///< [in] Calling thread's input item + T &output, ///< [out] Calling thread's output item (may be aliased to \p input) + ScanOp scan_op, ///< [in] Binary scan operator + T &block_aggregate) ///< [out] Threadblock-wide aggregate reduction of input items + { + if (WARP_SYNCHRONOUS) + { + // Short-circuit directly to warp scan + WarpScan(temp_storage.warp_scan, 0, linear_tid).InclusiveScan( + input, + output, + scan_op, + block_aggregate); + } + else + { + // Place thread partial into shared memory raking grid + T *placement_ptr = BlockRakingLayout::PlacementPtr(temp_storage.raking_grid, linear_tid); + *placement_ptr = input; + + __syncthreads(); + + // Reduce parallelism down to just raking threads + if (linear_tid < RAKING_THREADS) + { + // Raking upsweep reduction in grid + T raking_partial = Upsweep(scan_op); + + // Exclusive warp synchronous scan + WarpScan(temp_storage.warp_scan, 0, linear_tid).ExclusiveScan( + raking_partial, + raking_partial, + scan_op, + temp_storage.block_aggregate); + + // Inclusive raking downsweep scan + InclusiveDownsweep(scan_op, raking_partial, (linear_tid != 0)); + } + + __syncthreads(); + + // Grab thread prefix from shared memory + output = *placement_ptr; + + // Retrieve block aggregate + block_aggregate = temp_storage.block_aggregate; + } + } + + + /// Computes an inclusive threadblock-wide prefix scan using the specified binary \p scan_op functor. Each thread contributes one input element. the call-back functor \p block_prefix_op is invoked by the first warp in the block, and the value returned by lane0 in that warp is used as the "seed" value that logically prefixes the threadblock's scan inputs. Also provides every thread with the block-wide \p block_aggregate of all inputs. + template < + typename ScanOp, + typename BlockPrefixOp> + __device__ __forceinline__ void InclusiveScan( + T input, ///< [in] Calling thread's input item + T &output, ///< [out] Calling thread's output item (may be aliased to \p input) + ScanOp scan_op, ///< [in] Binary scan operator + T &block_aggregate, ///< [out] Threadblock-wide aggregate reduction of input items (exclusive of the \p block_prefix_op value) + BlockPrefixOp &block_prefix_op) ///< [in-out] [warp0 only] Call-back functor for specifying a threadblock-wide prefix to be applied to all inputs. + { + if (WARP_SYNCHRONOUS) + { + // Short-circuit directly to warp scan + WarpScan(temp_storage.warp_scan, 0, linear_tid).InclusiveScan( + input, + output, + scan_op, + block_aggregate, + block_prefix_op); + } + else + { + // Place thread partial into shared memory raking grid + T *placement_ptr = BlockRakingLayout::PlacementPtr(temp_storage.raking_grid, linear_tid); + *placement_ptr = input; + + __syncthreads(); + + // Reduce parallelism down to just raking threads + if (linear_tid < RAKING_THREADS) + { + // Raking upsweep reduction in grid + T raking_partial = Upsweep(scan_op); + + // Warp synchronous scan + WarpScan(temp_storage.warp_scan, 0, linear_tid).ExclusiveScan( + raking_partial, + raking_partial, + scan_op, + temp_storage.block_aggregate, + block_prefix_op); + + // Inclusive raking downsweep scan + InclusiveDownsweep(scan_op, raking_partial); + } + + __syncthreads(); + + // Grab thread prefix from shared memory + output = *placement_ptr; + + // Retrieve block aggregate + block_aggregate = temp_storage.block_aggregate; + } + } + + + /// Computes an inclusive threadblock-wide prefix scan using the specified binary \p scan_op functor. Each thread contributes one input element. Also provides every thread with the block-wide \p block_aggregate of all inputs. + __device__ __forceinline__ void InclusiveSum( + T input, ///< [in] Calling thread's input item + T &output, ///< [out] Calling thread's output item (may be aliased to \p input) + T &block_aggregate) ///< [out] Threadblock-wide aggregate reduction of input items + { + if (WARP_SYNCHRONOUS) + { + // Short-circuit directly to warp scan + WarpScan(temp_storage.warp_scan, 0, linear_tid).InclusiveSum( + input, + output, + block_aggregate); + } + else + { + // Raking scan + Sum scan_op; + + // Place thread partial into shared memory raking grid + T *placement_ptr = BlockRakingLayout::PlacementPtr(temp_storage.raking_grid, linear_tid); + *placement_ptr = input; + + __syncthreads(); + + // Reduce parallelism down to just raking threads + if (linear_tid < RAKING_THREADS) + { + // Raking upsweep reduction in grid + T raking_partial = Upsweep(scan_op); + + // Exclusive warp synchronous scan + WarpScan(temp_storage.warp_scan, 0, linear_tid).ExclusiveSum( + raking_partial, + raking_partial, + temp_storage.block_aggregate); + + // Inclusive raking downsweep scan + InclusiveDownsweep(scan_op, raking_partial, (linear_tid != 0)); + } + + __syncthreads(); + + // Grab thread prefix from shared memory + output = *placement_ptr; + + // Retrieve block aggregate + block_aggregate = temp_storage.block_aggregate; + } + } + + + /// Computes an inclusive threadblock-wide prefix scan using the specified binary \p scan_op functor. Each thread contributes one input element. Instead of using 0 as the threadblock-wide prefix, the call-back functor \p block_prefix_op is invoked by the first warp in the block, and the value returned by lane0 in that warp is used as the "seed" value that logically prefixes the threadblock's scan inputs. Also provides every thread with the block-wide \p block_aggregate of all inputs. + template + __device__ __forceinline__ void InclusiveSum( + T input, ///< [in] Calling thread's input item + T &output, ///< [out] Calling thread's output item (may be aliased to \p input) + T &block_aggregate, ///< [out] Threadblock-wide aggregate reduction of input items (exclusive of the \p block_prefix_op value) + BlockPrefixOp &block_prefix_op) ///< [in-out] [warp0 only] Call-back functor for specifying a threadblock-wide prefix to be applied to all inputs. + { + if (WARP_SYNCHRONOUS) + { + // Short-circuit directly to warp scan + WarpScan(temp_storage.warp_scan, 0, linear_tid).InclusiveSum( + input, + output, + block_aggregate, + block_prefix_op); + } + else + { + // Raking scan + Sum scan_op; + + // Place thread partial into shared memory raking grid + T *placement_ptr = BlockRakingLayout::PlacementPtr(temp_storage.raking_grid, linear_tid); + *placement_ptr = input; + + __syncthreads(); + + // Reduce parallelism down to just raking threads + if (linear_tid < RAKING_THREADS) + { + // Raking upsweep reduction in grid + T raking_partial = Upsweep(scan_op); + + // Warp synchronous scan + WarpScan(temp_storage.warp_scan, 0, linear_tid).ExclusiveSum( + raking_partial, + raking_partial, + temp_storage.block_aggregate, + block_prefix_op); + + // Inclusive raking downsweep scan + InclusiveDownsweep(scan_op, raking_partial); + } + + __syncthreads(); + + // Grab thread prefix from shared memory + output = *placement_ptr; + + // Retrieve block aggregate + block_aggregate = temp_storage.block_aggregate; + } + } + +}; + + +} // CUB namespace +CUB_NS_POSTFIX // Optional outer namespace(s) + diff --git a/lib/kokkos/TPL/cub/block/specializations/block_scan_warp_scans.cuh b/lib/kokkos/TPL/cub/block/specializations/block_scan_warp_scans.cuh new file mode 100755 index 0000000000..f7af3613de --- /dev/null +++ b/lib/kokkos/TPL/cub/block/specializations/block_scan_warp_scans.cuh @@ -0,0 +1,342 @@ +/****************************************************************************** + * Copyright (c) 2011, Duane Merrill. All rights reserved. + * Copyright (c) 2011-2013, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the NVIDIA CORPORATION nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + ******************************************************************************/ + +/** + * \file + * cub::BlockScanWarpscans provides warpscan-based variants of parallel prefix scan across a CUDA threadblock. + */ + +#pragma once + +#include "../../util_arch.cuh" +#include "../../warp/warp_scan.cuh" +#include "../../util_namespace.cuh" + +/// Optional outer namespace(s) +CUB_NS_PREFIX + +/// CUB namespace +namespace cub { + +/** + * \brief BlockScanWarpScans provides warpscan-based variants of parallel prefix scan across a CUDA threadblock. + */ +template < + typename T, + int BLOCK_THREADS> +struct BlockScanWarpScans +{ + /// Constants + enum + { + /// Number of active warps + WARPS = (BLOCK_THREADS + PtxArchProps::WARP_THREADS - 1) / PtxArchProps::WARP_THREADS, + }; + + /// WarpScan utility type + typedef WarpScan WarpScan; + + /// Shared memory storage layout type + struct _TempStorage + { + typename WarpScan::TempStorage warp_scan; ///< Buffer for warp-synchronous scan + T warp_aggregates[WARPS]; ///< Shared totals from each warp-synchronous scan + T block_prefix; ///< Shared prefix for the entire threadblock + }; + + + /// Alias wrapper allowing storage to be unioned + struct TempStorage : Uninitialized<_TempStorage> {}; + + + // Thread fields + _TempStorage &temp_storage; + int linear_tid; + int warp_id; + int lane_id; + + + /// Constructor + __device__ __forceinline__ BlockScanWarpScans( + TempStorage &temp_storage, + int linear_tid) + : + temp_storage(temp_storage.Alias()), + linear_tid(linear_tid), + warp_id((BLOCK_THREADS <= PtxArchProps::WARP_THREADS) ? + 0 : + linear_tid / PtxArchProps::WARP_THREADS), + lane_id((BLOCK_THREADS <= PtxArchProps::WARP_THREADS) ? + linear_tid : + linear_tid % PtxArchProps::WARP_THREADS) + {} + + + /// Update the calling thread's partial reduction with the warp-wide aggregates from preceding warps. Also returns block-wide aggregate in thread0. + template + __device__ __forceinline__ void ApplyWarpAggregates( + T &partial, ///< [out] The calling thread's partial reduction + ScanOp scan_op, ///< [in] Binary scan operator + T warp_aggregate, ///< [in] [lane0s only] Warp-wide aggregate reduction of input items + T &block_aggregate, ///< [out] Threadblock-wide aggregate reduction of input items + bool lane_valid = true) ///< [in] Whether or not the partial belonging to the current thread is valid + { + // Share lane aggregates + temp_storage.warp_aggregates[warp_id] = warp_aggregate; + + __syncthreads(); + + block_aggregate = temp_storage.warp_aggregates[0]; + + #pragma unroll + for (int WARP = 1; WARP < WARPS; WARP++) + { + if (warp_id == WARP) + { + partial = (lane_valid) ? + scan_op(block_aggregate, partial) : // fold it in our valid partial + block_aggregate; // replace our invalid partial with the aggregate + } + + block_aggregate = scan_op(block_aggregate, temp_storage.warp_aggregates[WARP]); + } + } + + + /// Computes an exclusive threadblock-wide prefix scan using the specified binary \p scan_op functor. Each thread contributes one input element. Also provides every thread with the block-wide \p block_aggregate of all inputs. + template + __device__ __forceinline__ void ExclusiveScan( + T input, ///< [in] Calling thread's input items + T &output, ///< [out] Calling thread's output items (may be aliased to \p input) + const T &identity, ///< [in] Identity value + ScanOp scan_op, ///< [in] Binary scan operator + T &block_aggregate) ///< [out] Threadblock-wide aggregate reduction of input items + { + T warp_aggregate; + WarpScan(temp_storage.warp_scan, warp_id, lane_id).ExclusiveScan(input, output, identity, scan_op, warp_aggregate); + + // Update outputs and block_aggregate with warp-wide aggregates + ApplyWarpAggregates(output, scan_op, warp_aggregate, block_aggregate); + } + + + /// Computes an exclusive threadblock-wide prefix scan using the specified binary \p scan_op functor. Each thread contributes one input element. the call-back functor \p block_prefix_op is invoked by the first warp in the block, and the value returned by lane0 in that warp is used as the "seed" value that logically prefixes the threadblock's scan inputs. Also provides every thread with the block-wide \p block_aggregate of all inputs. + template < + typename ScanOp, + typename BlockPrefixOp> + __device__ __forceinline__ void ExclusiveScan( + T input, ///< [in] Calling thread's input item + T &output, ///< [out] Calling thread's output item (may be aliased to \p input) + T identity, ///< [in] Identity value + ScanOp scan_op, ///< [in] Binary scan operator + T &block_aggregate, ///< [out] Threadblock-wide aggregate reduction of input items (exclusive of the \p block_prefix_op value) + BlockPrefixOp &block_prefix_op) ///< [in-out] [warp0 only] Call-back functor for specifying a threadblock-wide prefix to be applied to all inputs. + { + ExclusiveScan(input, output, identity, scan_op, block_aggregate); + + // Compute and share threadblock prefix + if (warp_id == 0) + { + temp_storage.block_prefix = block_prefix_op(block_aggregate); + } + + __syncthreads(); + + // Incorporate threadblock prefix into outputs + output = scan_op(temp_storage.block_prefix, output); + } + + + /// Computes an exclusive threadblock-wide prefix scan using the specified binary \p scan_op functor. Each thread contributes one input element. Also provides every thread with the block-wide \p block_aggregate of all inputs. With no identity value, the output computed for thread0 is undefined. + template + __device__ __forceinline__ void ExclusiveScan( + T input, ///< [in] Calling thread's input item + T &output, ///< [out] Calling thread's output item (may be aliased to \p input) + ScanOp scan_op, ///< [in] Binary scan operator + T &block_aggregate) ///< [out] Threadblock-wide aggregate reduction of input items + { + T warp_aggregate; + WarpScan(temp_storage.warp_scan, warp_id, lane_id).ExclusiveScan(input, output, scan_op, warp_aggregate); + + // Update outputs and block_aggregate with warp-wide aggregates + ApplyWarpAggregates(output, scan_op, warp_aggregate, block_aggregate, (lane_id > 0)); + } + + + /// Computes an exclusive threadblock-wide prefix scan using the specified binary \p scan_op functor. Each thread contributes one input element. the call-back functor \p block_prefix_op is invoked by the first warp in the block, and the value returned by lane0 in that warp is used as the "seed" value that logically prefixes the threadblock's scan inputs. Also provides every thread with the block-wide \p block_aggregate of all inputs. + template < + typename ScanOp, + typename BlockPrefixOp> + __device__ __forceinline__ void ExclusiveScan( + T input, ///< [in] Calling thread's input item + T &output, ///< [out] Calling thread's output item (may be aliased to \p input) + ScanOp scan_op, ///< [in] Binary scan operator + T &block_aggregate, ///< [out] Threadblock-wide aggregate reduction of input items (exclusive of the \p block_prefix_op value) + BlockPrefixOp &block_prefix_op) ///< [in-out] [warp0 only] Call-back functor for specifying a threadblock-wide prefix to be applied to all inputs. + { + ExclusiveScan(input, output, scan_op, block_aggregate); + + // Compute and share threadblock prefix + if (warp_id == 0) + { + temp_storage.block_prefix = block_prefix_op(block_aggregate); + } + + __syncthreads(); + + // Incorporate threadblock prefix into outputs + output = (linear_tid == 0) ? + temp_storage.block_prefix : + scan_op(temp_storage.block_prefix, output); + } + + + /// Computes an exclusive threadblock-wide prefix scan using addition (+) as the scan operator. Each thread contributes one input element. Also provides every thread with the block-wide \p block_aggregate of all inputs. + __device__ __forceinline__ void ExclusiveSum( + T input, ///< [in] Calling thread's input item + T &output, ///< [out] Calling thread's output item (may be aliased to \p input) + T &block_aggregate) ///< [out] Threadblock-wide aggregate reduction of input items + { + T warp_aggregate; + WarpScan(temp_storage.warp_scan, warp_id, lane_id).ExclusiveSum(input, output, warp_aggregate); + + // Update outputs and block_aggregate with warp-wide aggregates from lane-0s + ApplyWarpAggregates(output, Sum(), warp_aggregate, block_aggregate); + } + + + /// Computes an exclusive threadblock-wide prefix scan using addition (+) as the scan operator. Each thread contributes one input element. Instead of using 0 as the threadblock-wide prefix, the call-back functor \p block_prefix_op is invoked by the first warp in the block, and the value returned by lane0 in that warp is used as the "seed" value that logically prefixes the threadblock's scan inputs. Also provides every thread with the block-wide \p block_aggregate of all inputs. + template + __device__ __forceinline__ void ExclusiveSum( + T input, ///< [in] Calling thread's input item + T &output, ///< [out] Calling thread's output item (may be aliased to \p input) + T &block_aggregate, ///< [out] Threadblock-wide aggregate reduction of input items (exclusive of the \p block_prefix_op value) + BlockPrefixOp &block_prefix_op) ///< [in-out] [warp0 only] Call-back functor for specifying a threadblock-wide prefix to be applied to all inputs. + { + ExclusiveSum(input, output, block_aggregate); + + // Compute and share threadblock prefix + if (warp_id == 0) + { + temp_storage.block_prefix = block_prefix_op(block_aggregate); + } + + __syncthreads(); + + // Incorporate threadblock prefix into outputs + Sum scan_op; + output = scan_op(temp_storage.block_prefix, output); + } + + + /// Computes an inclusive threadblock-wide prefix scan using the specified binary \p scan_op functor. Each thread contributes one input element. Also provides every thread with the block-wide \p block_aggregate of all inputs. + template + __device__ __forceinline__ void InclusiveScan( + T input, ///< [in] Calling thread's input item + T &output, ///< [out] Calling thread's output item (may be aliased to \p input) + ScanOp scan_op, ///< [in] Binary scan operator + T &block_aggregate) ///< [out] Threadblock-wide aggregate reduction of input items + { + T warp_aggregate; + WarpScan(temp_storage.warp_scan, warp_id, lane_id).InclusiveScan(input, output, scan_op, warp_aggregate); + + // Update outputs and block_aggregate with warp-wide aggregates from lane-0s + ApplyWarpAggregates(output, scan_op, warp_aggregate, block_aggregate); + + } + + + /// Computes an inclusive threadblock-wide prefix scan using the specified binary \p scan_op functor. Each thread contributes one input element. the call-back functor \p block_prefix_op is invoked by the first warp in the block, and the value returned by lane0 in that warp is used as the "seed" value that logically prefixes the threadblock's scan inputs. Also provides every thread with the block-wide \p block_aggregate of all inputs. + template < + typename ScanOp, + typename BlockPrefixOp> + __device__ __forceinline__ void InclusiveScan( + T input, ///< [in] Calling thread's input item + T &output, ///< [out] Calling thread's output item (may be aliased to \p input) + ScanOp scan_op, ///< [in] Binary scan operator + T &block_aggregate, ///< [out] Threadblock-wide aggregate reduction of input items (exclusive of the \p block_prefix_op value) + BlockPrefixOp &block_prefix_op) ///< [in-out] [warp0 only] Call-back functor for specifying a threadblock-wide prefix to be applied to all inputs. + { + InclusiveScan(input, output, scan_op, block_aggregate); + + // Compute and share threadblock prefix + if (warp_id == 0) + { + temp_storage.block_prefix = block_prefix_op(block_aggregate); + } + + __syncthreads(); + + // Incorporate threadblock prefix into outputs + output = scan_op(temp_storage.block_prefix, output); + } + + + /// Computes an inclusive threadblock-wide prefix scan using the specified binary \p scan_op functor. Each thread contributes one input element. Also provides every thread with the block-wide \p block_aggregate of all inputs. + __device__ __forceinline__ void InclusiveSum( + T input, ///< [in] Calling thread's input item + T &output, ///< [out] Calling thread's output item (may be aliased to \p input) + T &block_aggregate) ///< [out] Threadblock-wide aggregate reduction of input items + { + T warp_aggregate; + WarpScan(temp_storage.warp_scan, warp_id, lane_id).InclusiveSum(input, output, warp_aggregate); + + // Update outputs and block_aggregate with warp-wide aggregates from lane-0s + ApplyWarpAggregates(output, Sum(), warp_aggregate, block_aggregate); + } + + + /// Computes an inclusive threadblock-wide prefix scan using the specified binary \p scan_op functor. Each thread contributes one input element. Instead of using 0 as the threadblock-wide prefix, the call-back functor \p block_prefix_op is invoked by the first warp in the block, and the value returned by lane0 in that warp is used as the "seed" value that logically prefixes the threadblock's scan inputs. Also provides every thread with the block-wide \p block_aggregate of all inputs. + template + __device__ __forceinline__ void InclusiveSum( + T input, ///< [in] Calling thread's input item + T &output, ///< [out] Calling thread's output item (may be aliased to \p input) + T &block_aggregate, ///< [out] Threadblock-wide aggregate reduction of input items (exclusive of the \p block_prefix_op value) + BlockPrefixOp &block_prefix_op) ///< [in-out] [warp0 only] Call-back functor for specifying a threadblock-wide prefix to be applied to all inputs. + { + InclusiveSum(input, output, block_aggregate); + + // Compute and share threadblock prefix + if (warp_id == 0) + { + temp_storage.block_prefix = block_prefix_op(block_aggregate); + } + + __syncthreads(); + + // Incorporate threadblock prefix into outputs + Sum scan_op; + output = scan_op(temp_storage.block_prefix, output); + } + +}; + + +} // CUB namespace +CUB_NS_POSTFIX // Optional outer namespace(s) + diff --git a/lib/kokkos/TPL/cub/cub.cuh b/lib/kokkos/TPL/cub/cub.cuh new file mode 100755 index 0000000000..dbb77da225 --- /dev/null +++ b/lib/kokkos/TPL/cub/cub.cuh @@ -0,0 +1,84 @@ +/****************************************************************************** + * Copyright (c) 2011, Duane Merrill. All rights reserved. + * Copyright (c) 2011-2013, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the NVIDIA CORPORATION nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + ******************************************************************************/ + +/** + * \file + * CUB umbrella include file + */ + +#pragma once + + +// Block +#include "block/block_histogram.cuh" +#include "block/block_discontinuity.cuh" +#include "block/block_exchange.cuh" +#include "block/block_load.cuh" +#include "block/block_radix_rank.cuh" +#include "block/block_radix_sort.cuh" +#include "block/block_reduce.cuh" +#include "block/block_scan.cuh" +#include "block/block_store.cuh" + +// Device +#include "device/device_histogram.cuh" +#include "device/device_radix_sort.cuh" +#include "device/device_reduce.cuh" +#include "device/device_scan.cuh" + +// Grid +//#include "grid/grid_barrier.cuh" +#include "grid/grid_even_share.cuh" +#include "grid/grid_mapping.cuh" +#include "grid/grid_queue.cuh" + +// Host +#include "host/spinlock.cuh" + +// Thread +#include "thread/thread_load.cuh" +#include "thread/thread_operators.cuh" +#include "thread/thread_reduce.cuh" +#include "thread/thread_scan.cuh" +#include "thread/thread_store.cuh" + +// Warp +#include "warp/warp_reduce.cuh" +#include "warp/warp_scan.cuh" + +// Util +#include "util_allocator.cuh" +#include "util_arch.cuh" +#include "util_debug.cuh" +#include "util_device.cuh" +#include "util_macro.cuh" +#include "util_ptx.cuh" +#include "util_type.cuh" +#include "util_iterator.cuh" +#include "util_vector.cuh" + diff --git a/lib/kokkos/TPL/cub/device/block/block_histo_tiles.cuh b/lib/kokkos/TPL/cub/device/block/block_histo_tiles.cuh new file mode 100755 index 0000000000..e1165d60c3 --- /dev/null +++ b/lib/kokkos/TPL/cub/device/block/block_histo_tiles.cuh @@ -0,0 +1,322 @@ +/****************************************************************************** + * Copyright (c) 2011, Duane Merrill. All rights reserved. + * Copyright (c) 2011-2013, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the NVIDIA CORPORATION nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + ******************************************************************************/ + +/** + * \file + * cub::BlockHistogramTiles implements a stateful abstraction of CUDA thread blocks for histogramming multiple tiles as part of device-wide histogram. + */ + +#pragma once + +#include + +#include "specializations/block_histo_tiles_gatomic.cuh" +#include "specializations/block_histo_tiles_satomic.cuh" +#include "specializations/block_histo_tiles_sort.cuh" +#include "../../util_type.cuh" +#include "../../grid/grid_mapping.cuh" +#include "../../grid/grid_even_share.cuh" +#include "../../grid/grid_queue.cuh" +#include "../../util_namespace.cuh" + +/// Optional outer namespace(s) +CUB_NS_PREFIX + +/// CUB namespace +namespace cub { + + +/****************************************************************************** + * Algorithmic variants + ******************************************************************************/ + + +/** + * \brief BlockHistogramTilesAlgorithm enumerates alternative algorithms for BlockHistogramTiles. + */ +enum BlockHistogramTilesAlgorithm +{ + + /** + * \par Overview + * A two-kernel approach in which: + * -# Thread blocks in the first kernel aggregate their own privatized + * histograms using block-wide sorting (see BlockHistogramAlgorithm::BLOCK_HISTO_SORT). + * -# A single thread block in the second kernel reduces them into the output histogram(s). + * + * \par Performance Considerations + * Delivers consistent throughput regardless of sample bin distribution. + * + * However, because histograms are privatized in shared memory, a large + * number of bins (e.g., thousands) may adversely affect occupancy and + * performance (or even the ability to launch). + */ + GRID_HISTO_SORT, + + + /** + * \par Overview + * A two-kernel approach in which: + * -# Thread blocks in the first kernel aggregate their own privatized + * histograms using shared-memory \p atomicAdd(). + * -# A single thread block in the second kernel reduces them into the + * output histogram(s). + * + * \par Performance Considerations + * Performance is strongly tied to the hardware implementation of atomic + * addition, and may be significantly degraded for non uniformly-random + * input distributions where many concurrent updates are likely to be + * made to the same bin counter. + * + * However, because histograms are privatized in shared memory, a large + * number of bins (e.g., thousands) may adversely affect occupancy and + * performance (or even the ability to launch). + */ + GRID_HISTO_SHARED_ATOMIC, + + + /** + * \par Overview + * A single-kernel approach in which thread blocks update the output histogram(s) directly + * using global-memory \p atomicAdd(). + * + * \par Performance Considerations + * Performance is strongly tied to the hardware implementation of atomic + * addition, and may be significantly degraded for non uniformly-random + * input distributions where many concurrent updates are likely to be + * made to the same bin counter. + * + * Performance is not significantly impacted when computing histograms having large + * numbers of bins (e.g., thousands). + */ + GRID_HISTO_GLOBAL_ATOMIC, + +}; + + +/****************************************************************************** + * Tuning policy + ******************************************************************************/ + +/** + * Tuning policy for BlockHistogramTiles + */ +template < + int _BLOCK_THREADS, + int _ITEMS_PER_THREAD, + BlockHistogramTilesAlgorithm _GRID_ALGORITHM, + GridMappingStrategy _GRID_MAPPING, + int _SM_OCCUPANCY> +struct BlockHistogramTilesPolicy +{ + enum + { + BLOCK_THREADS = _BLOCK_THREADS, + ITEMS_PER_THREAD = _ITEMS_PER_THREAD, + SM_OCCUPANCY = _SM_OCCUPANCY, + }; + + static const BlockHistogramTilesAlgorithm GRID_ALGORITHM = _GRID_ALGORITHM; + static const GridMappingStrategy GRID_MAPPING = _GRID_MAPPING; +}; + + + +/****************************************************************************** + * Thread block abstractions + ******************************************************************************/ + + +/** + * Implements a stateful abstraction of CUDA thread blocks for histogramming multiple tiles as part of device-wide histogram using global atomics + */ +template < + typename BlockHistogramTilesPolicy, ///< Tuning policy + int BINS, ///< Number of histogram bins per channel + int CHANNELS, ///< Number of channels interleaved in the input data (may be greater than the number of active channels being histogrammed) + int ACTIVE_CHANNELS, ///< Number of channels actively being histogrammed + typename InputIteratorRA, ///< The input iterator type (may be a simple pointer type). Must have a value type that can be cast as an integer in the range [0..BINS-1] + typename HistoCounter, ///< Integral type for counting sample occurrences per histogram bin + typename SizeT> ///< Integer type for offsets +struct BlockHistogramTiles +{ + //--------------------------------------------------------------------- + // Types and constants + //--------------------------------------------------------------------- + + // Histogram grid algorithm + static const BlockHistogramTilesAlgorithm GRID_ALGORITHM = BlockHistogramTilesPolicy::GRID_ALGORITHM; + + // Alternative internal implementation types + typedef BlockHistogramTilesSort< BlockHistogramTilesPolicy, BINS, CHANNELS, ACTIVE_CHANNELS, InputIteratorRA, HistoCounter, SizeT> BlockHistogramTilesSortT; + typedef BlockHistogramTilesSharedAtomic< BlockHistogramTilesPolicy, BINS, CHANNELS, ACTIVE_CHANNELS, InputIteratorRA, HistoCounter, SizeT> BlockHistogramTilesSharedAtomicT; + typedef BlockHistogramTilesGlobalAtomic< BlockHistogramTilesPolicy, BINS, CHANNELS, ACTIVE_CHANNELS, InputIteratorRA, HistoCounter, SizeT> BlockHistogramTilesGlobalAtomicT; + + // Internal block sweep histogram type + typedef typename If<(GRID_ALGORITHM == GRID_HISTO_SORT), + BlockHistogramTilesSortT, + typename If<(GRID_ALGORITHM == GRID_HISTO_SHARED_ATOMIC), + BlockHistogramTilesSharedAtomicT, + BlockHistogramTilesGlobalAtomicT>::Type>::Type InternalBlockDelegate; + + enum + { + TILE_ITEMS = InternalBlockDelegate::TILE_ITEMS, + }; + + + // Temporary storage type + typedef typename InternalBlockDelegate::TempStorage TempStorage; + + //--------------------------------------------------------------------- + // Per-thread fields + //--------------------------------------------------------------------- + + // Internal block delegate + InternalBlockDelegate internal_delegate; + + + //--------------------------------------------------------------------- + // Interface + //--------------------------------------------------------------------- + + /** + * Constructor + */ + __device__ __forceinline__ BlockHistogramTiles( + TempStorage &temp_storage, ///< Reference to temp_storage + InputIteratorRA d_in, ///< Input data to reduce + HistoCounter* (&d_out_histograms)[ACTIVE_CHANNELS]) ///< Reference to output histograms + : + internal_delegate(temp_storage, d_in, d_out_histograms) + {} + + + /** + * \brief Reduce a consecutive segment of input tiles + */ + __device__ __forceinline__ void ConsumeTiles( + SizeT block_offset, ///< [in] Threadblock begin offset (inclusive) + SizeT block_oob) ///< [in] Threadblock end offset (exclusive) + { + // Consume subsequent full tiles of input + while (block_offset + TILE_ITEMS <= block_oob) + { + internal_delegate.ConsumeTile(block_offset); + block_offset += TILE_ITEMS; + } + + // Consume a partially-full tile + if (block_offset < block_oob) + { + int valid_items = block_oob - block_offset; + internal_delegate.ConsumeTile(block_offset, valid_items); + } + + // Aggregate output + internal_delegate.AggregateOutput(); + } + + + /** + * Reduce a consecutive segment of input tiles + */ + __device__ __forceinline__ void ConsumeTiles( + SizeT num_items, ///< [in] Total number of global input items + GridEvenShare &even_share, ///< [in] GridEvenShare descriptor + GridQueue &queue, ///< [in,out] GridQueue descriptor + Int2Type is_even_share) ///< [in] Marker type indicating this is an even-share mapping + { + even_share.BlockInit(); + ConsumeTiles(even_share.block_offset, even_share.block_oob); + } + + + /** + * Dequeue and reduce tiles of items as part of a inter-block scan + */ + __device__ __forceinline__ void ConsumeTiles( + int num_items, ///< Total number of input items + GridQueue queue) ///< Queue descriptor for assigning tiles of work to thread blocks + { + // Shared block offset + __shared__ SizeT shared_block_offset; + + // We give each thread block at least one tile of input. + SizeT block_offset = blockIdx.x * TILE_ITEMS; + SizeT even_share_base = gridDim.x * TILE_ITEMS; + + // Process full tiles of input + while (block_offset + TILE_ITEMS <= num_items) + { + internal_delegate.ConsumeTile(block_offset); + + // Dequeue up to TILE_ITEMS + if (threadIdx.x == 0) + shared_block_offset = queue.Drain(TILE_ITEMS) + even_share_base; + + __syncthreads(); + + block_offset = shared_block_offset; + + __syncthreads(); + } + + // Consume a partially-full tile + if (block_offset < num_items) + { + int valid_items = num_items - block_offset; + internal_delegate.ConsumeTile(block_offset, valid_items); + } + + // Aggregate output + internal_delegate.AggregateOutput(); + } + + + /** + * Dequeue and reduce tiles of items as part of a inter-block scan + */ + __device__ __forceinline__ void ConsumeTiles( + SizeT num_items, ///< [in] Total number of global input items + GridEvenShare &even_share, ///< [in] GridEvenShare descriptor + GridQueue &queue, ///< [in,out] GridQueue descriptor + Int2Type is_dynamic) ///< [in] Marker type indicating this is a dynamic mapping + { + ConsumeTiles(num_items, queue); + } + + +}; + + + + +} // CUB namespace +CUB_NS_POSTFIX // Optional outer namespace(s) + diff --git a/lib/kokkos/TPL/cub/device/block/block_partition_tiles.cuh b/lib/kokkos/TPL/cub/device/block/block_partition_tiles.cuh new file mode 100755 index 0000000000..4597773af6 --- /dev/null +++ b/lib/kokkos/TPL/cub/device/block/block_partition_tiles.cuh @@ -0,0 +1,381 @@ +/****************************************************************************** + * Copyright (c) 2011, Duane Merrill. All rights reserved. + * Copyright (c) 2011-2013, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the NVIDIA CORPORATION nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + ******************************************************************************/ + +/** + * \file + * cub::BlockPartitionTiles implements a stateful abstraction of CUDA thread blocks for participating in device-wide list partitioning. + */ + +#pragma once + +#include + +#include "scan_tiles_types.cuh" +#include "../../thread/thread_operators.cuh" +#include "../../block/block_load.cuh" +#include "../../block/block_store.cuh" +#include "../../block/block_scan.cuh" +#include "../../grid/grid_queue.cuh" +#include "../../util_vector.cuh" +#include "../../util_namespace.cuh" + +/// Optional outer namespace(s) +CUB_NS_PREFIX + +/// CUB namespace +namespace cub { + + +/****************************************************************************** + * Tuning policy types + ******************************************************************************/ + +/** + * Tuning policy for BlockPartitionTiles + */ +template < + int _PARTITIONS, + int _BLOCK_THREADS, + int _ITEMS_PER_THREAD, + PtxLoadModifier _LOAD_MODIFIER, + BlockScanAlgorithm _SCAN_ALGORITHM> +struct BlockPartitionTilesPolicy +{ + enum + { + PARTITIONS = _PARTITIONS, + BLOCK_THREADS = _BLOCK_THREADS, + ITEMS_PER_THREAD = _ITEMS_PER_THREAD, + }; + + static const PtxLoadModifier LOAD_MODIFIER = _LOAD_MODIFIER; + static const BlockScanAlgorithm SCAN_ALGORITHM = _SCAN_ALGORITHM; +}; + + + +/** + * Tuple type for scanning partition membership flags + */ +template < + typename SizeT, + int PARTITIONS> +struct PartitionScanTuple; + + +/** + * Tuple type for scanning partition membership flags (specialized for 1 output partition) + */ +template +struct PartitionScanTuple : VectorHelper::Type +{ + __device__ __forceinline__ PartitionScanTuple operator+(const PartitionScanTuple &other) + { + PartitionScanTuple retval; + retval.x = x + other.x; + return retval; + } + + template + __device__ __forceinline__ void SetFlags(PredicateOp pred_op, T val) + { + this->x = pred_op(val); + } + + template + __device__ __forceinline__ void Scatter(PredicateOp pred_op, T val, OutputIteratorRA d_out, SizeT num_items) + { + if (pred_op(val)) + d_out[this->x - 1] = val; + } + +}; + + +/** + * Tuple type for scanning partition membership flags (specialized for 2 output partitions) + */ +template +struct PartitionScanTuple : VectorHelper::Type +{ + __device__ __forceinline__ PartitionScanTuple operator+(const PartitionScanTuple &other) + { + PartitionScanTuple retval; + retval.x = x + other.x; + retval.y = y + other.y; + return retval; + } + + template + __device__ __forceinline__ void SetFlags(PredicateOp pred_op, T val) + { + bool pred = pred_op(val); + this->x = pred; + this->y = !pred; + } + + template + __device__ __forceinline__ void Scatter(PredicateOp pred_op, T val, OutputIteratorRA d_out, SizeT num_items) + { + SizeT scatter_offset = (pred_op(val)) ? + this->x - 1 : + num_items - this->y; + + d_out[scatter_offset] = val; + } +}; + + + + +/****************************************************************************** + * Thread block abstractions + ******************************************************************************/ + +/** + * \brief BlockPartitionTiles implements a stateful abstraction of CUDA thread blocks for participating in device-wide list partitioning. + * + * Implements a single-pass "domino" strategy with adaptive prefix lookback. + */ +template < + typename BlockPartitionTilesPolicy, ///< Tuning policy + typename InputIteratorRA, ///< Input iterator type + typename OutputIteratorRA, ///< Output iterator type + typename PredicateOp, ///< Partition predicate functor type + typename SizeT> ///< Offset integer type +struct BlockPartitionTiles +{ + //--------------------------------------------------------------------- + // Types and constants + //--------------------------------------------------------------------- + + // Constants + enum + { + PARTITIONS = BlockPartitionTilesPolicy::PARTITIONS, + BLOCK_THREADS = BlockPartitionTilesPolicy::BLOCK_THREADS, + ITEMS_PER_THREAD = BlockPartitionTilesPolicy::ITEMS_PER_THREAD, + TILE_ITEMS = BLOCK_THREADS * ITEMS_PER_THREAD, + }; + + // Load modifier + static const PtxLoadModifier LOAD_MODIFIER = BlockPartitionTilesPolicy::LOAD_MODIFIER; + + // Data type of input iterator + typedef typename std::iterator_traits::value_type T; + + // Tuple type for scanning partition membership flags + typedef PartitionScanTuple PartitionScanTuple; + + // Tile status descriptor type + typedef ScanTileDescriptor ScanTileDescriptorT; + + // Block scan type for scanning membership flag scan_tuples + typedef BlockScan< + PartitionScanTuple, + BlockPartitionTilesPolicy::BLOCK_THREADS, + BlockPartitionTilesPolicy::SCAN_ALGORITHM> BlockScanT; + + // Callback type for obtaining inter-tile prefix during block scan + typedef DeviceScanBlockPrefixOp InterblockPrefixOp; + + // Shared memory type for this threadblock + struct TempStorage + { + typename InterblockPrefixOp::TempStorage prefix; // Smem needed for cooperative prefix callback + typename BlockScanT::TempStorage scan; // Smem needed for tile scanning + SizeT tile_idx; // Shared tile index + }; + + + //--------------------------------------------------------------------- + // Per-thread fields + //--------------------------------------------------------------------- + + TempStorage &temp_storage; ///< Reference to temp_storage + InputIteratorRA d_in; ///< Input data + OutputIteratorRA d_out; ///< Output data + ScanTileDescriptorT *d_tile_status; ///< Global list of tile status + PredicateOp pred_op; ///< Unary predicate operator indicating membership in the first partition + SizeT num_items; ///< Total number of input items + + + //--------------------------------------------------------------------- + // Constructor + //--------------------------------------------------------------------- + + // Constructor + __device__ __forceinline__ + BlockPartitionTiles( + TempStorage &temp_storage, ///< Reference to temp_storage + InputIteratorRA d_in, ///< Input data + OutputIteratorRA d_out, ///< Output data + ScanTileDescriptorT *d_tile_status, ///< Global list of tile status + PredicateOp pred_op, ///< Unary predicate operator indicating membership in the first partition + SizeT num_items) ///< Total number of input items + : + temp_storage(temp_storage.Alias()), + d_in(d_in), + d_out(d_out), + d_tile_status(d_tile_status), + pred_op(pred_op), + num_items(num_items) + {} + + + //--------------------------------------------------------------------- + // Domino scan + //--------------------------------------------------------------------- + + /** + * Process a tile of input + */ + template + __device__ __forceinline__ void ConsumeTile( + int tile_idx, ///< Tile index + SizeT block_offset, ///< Tile offset + PartitionScanTuple &partition_ends) ///< Running total + { + T items[ITEMS_PER_THREAD]; + PartitionScanTuple scan_tuples[ITEMS_PER_THREAD]; + + // Load items + int valid_items = num_items - block_offset; + if (FULL_TILE) + LoadStriped(threadIdx.x, d_in + block_offset, items); + else + LoadStriped(threadIdx.x, d_in + block_offset, items, valid_items); + + // Prevent hoisting +// __syncthreads(); +// __threadfence_block(); + + // Set partition membership flags in scan scan_tuples + #pragma unroll + for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ++ITEM) + { + scan_tuples[ITEM].SetFlags(pred_op, items[ITEM]); + } + + // Perform inclusive scan over scan scan_tuples + PartitionScanTuple block_aggregate; + if (tile_idx == 0) + { + BlockScanT(temp_storage.scan).InclusiveScan(scan_tuples, scan_tuples, Sum(), block_aggregate); + partition_ends = block_aggregate; + + // Update tile status if there are successor tiles + if (FULL_TILE && (threadIdx.x == 0)) + ScanTileDescriptorT::SetPrefix(d_tile_status, block_aggregate); + } + else + { + InterblockPrefixOp prefix_op(d_tile_status, temp_storage.prefix, Sum(), tile_idx); + BlockScanT(temp_storage.scan).InclusiveScan(scan_tuples, scan_tuples, Sum(), block_aggregate, prefix_op); + partition_ends = prefix_op.inclusive_prefix; + } + + // Scatter items + #pragma unroll + for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ++ITEM) + { + // Scatter if not out-of-bounds + if (FULL_TILE || (threadIdx.x + (ITEM * BLOCK_THREADS) < valid_items)) + { + scan_tuples[ITEM].Scatter(pred_op, items[ITEM], d_out, num_items); + } + } + } + + + /** + * Dequeue and scan tiles of items as part of a domino scan + */ + __device__ __forceinline__ void ConsumeTiles( + GridQueue queue, ///< [in] Queue descriptor for assigning tiles of work to thread blocks + SizeT num_tiles, ///< [in] Total number of input tiles + PartitionScanTuple &partition_ends, ///< [out] Running partition end offsets + bool &is_last_tile) ///< [out] Whether or not this block handled the last tile (i.e., partition_ends is valid for the entire input) + { +#if CUB_PTX_ARCH < 200 + + // No concurrent kernels allowed and blocks are launched in increasing order, so just assign one tile per block (up to 65K blocks) + int tile_idx = blockIdx.x; + SizeT block_offset = SizeT(TILE_ITEMS) * tile_idx; + + if (block_offset + TILE_ITEMS <= num_items) + { + ConsumeTile(tile_idx, block_offset, partition_ends); + } + else if (block_offset < num_items) + { + ConsumeTile(tile_idx, block_offset, partition_ends); + } + is_last_tile = (tile_idx == num_tiles - 1); + +#else + + // Get first tile + if (threadIdx.x == 0) + temp_storage.tile_idx = queue.Drain(1); + + __syncthreads(); + + int tile_idx = temp_storage.tile_idx; + SizeT block_offset = SizeT(TILE_ITEMS) * tile_idx; + + while (block_offset + TILE_ITEMS <= num_items) + { + // Consume full tile + ConsumeTile(tile_idx, block_offset, partition_ends); + is_last_tile = (tile_idx == num_tiles - 1); + + // Get next tile + if (threadIdx.x == 0) + temp_storage.tile_idx = queue.Drain(1); + + __syncthreads(); + + tile_idx = temp_storage.tile_idx; + block_offset = SizeT(TILE_ITEMS) * tile_idx; + } + + // Consume a partially-full tile + if (block_offset < num_items) + { + ConsumeTile(tile_idx, block_offset, partition_ends); + is_last_tile = (tile_idx == num_tiles - 1); + } +#endif + } +}; + + +} // CUB namespace +CUB_NS_POSTFIX // Optional outer namespace(s) + diff --git a/lib/kokkos/TPL/cub/device/block/block_radix_sort_downsweep_tiles.cuh b/lib/kokkos/TPL/cub/device/block/block_radix_sort_downsweep_tiles.cuh new file mode 100755 index 0000000000..91d628e000 --- /dev/null +++ b/lib/kokkos/TPL/cub/device/block/block_radix_sort_downsweep_tiles.cuh @@ -0,0 +1,713 @@ +/****************************************************************************** + * Copyright (c) 2011, Duane Merrill. All rights reserved. + * Copyright (c) 2011-2013, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the NVIDIA CORPORATION nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + ******************************************************************************/ + +/** + * \file + * BlockRadixSortDownsweepTiles implements a stateful abstraction of CUDA thread blocks for participating in device-wide radix sort downsweep. + */ + + +#pragma once + +#include "../../thread/thread_load.cuh" +#include "../../block/block_load.cuh" +#include "../../block/block_store.cuh" +#include "../../block/block_radix_rank.cuh" +#include "../../block/block_exchange.cuh" +#include "../../util_type.cuh" +#include "../../util_namespace.cuh" + +/// Optional outer namespace(s) +CUB_NS_PREFIX + +/// CUB namespace +namespace cub { + + +/****************************************************************************** + * Tuning policy types + ******************************************************************************/ + +/** + * Types of scattering strategies + */ +enum RadixSortScatterAlgorithm +{ + RADIX_SORT_SCATTER_DIRECT, ///< Scatter directly from registers to global bins + RADIX_SORT_SCATTER_TWO_PHASE, ///< First scatter from registers into shared memory bins, then into global bins +}; + + +/** + * Tuning policy for BlockRadixSortDownsweepTiles + */ +template < + int _BLOCK_THREADS, ///< The number of threads per CTA + int _ITEMS_PER_THREAD, ///< The number of consecutive downsweep keys to process per thread + BlockLoadAlgorithm _LOAD_ALGORITHM, ///< The BlockLoad algorithm to use + PtxLoadModifier _LOAD_MODIFIER, ///< The PTX cache-modifier to use for loads + bool _EXCHANGE_TIME_SLICING, ///< Whether or not to time-slice key/value exchanges through shared memory to lower shared memory pressure + bool _MEMOIZE_OUTER_SCAN, ///< Whether or not to buffer outer raking scan partials to incur fewer shared memory reads at the expense of higher register pressure. See BlockScanAlgorithm::BLOCK_SCAN_RAKING_MEMOIZE for more details. + BlockScanAlgorithm _INNER_SCAN_ALGORITHM, ///< The cub::BlockScanAlgorithm algorithm to use + RadixSortScatterAlgorithm _SCATTER_ALGORITHM, ///< The scattering strategy to use + cudaSharedMemConfig _SMEM_CONFIG, ///< Shared memory bank mode (default: \p cudaSharedMemBankSizeFourByte) + int _RADIX_BITS> ///< The number of radix bits, i.e., log2(bins) +struct BlockRadixSortDownsweepTilesPolicy +{ + enum + { + BLOCK_THREADS = _BLOCK_THREADS, + ITEMS_PER_THREAD = _ITEMS_PER_THREAD, + EXCHANGE_TIME_SLICING = _EXCHANGE_TIME_SLICING, + RADIX_BITS = _RADIX_BITS, + MEMOIZE_OUTER_SCAN = _MEMOIZE_OUTER_SCAN, + TILE_ITEMS = BLOCK_THREADS * ITEMS_PER_THREAD, + }; + + static const BlockLoadAlgorithm LOAD_ALGORITHM = _LOAD_ALGORITHM; + static const PtxLoadModifier LOAD_MODIFIER = _LOAD_MODIFIER; + static const BlockScanAlgorithm INNER_SCAN_ALGORITHM = _INNER_SCAN_ALGORITHM; + static const RadixSortScatterAlgorithm SCATTER_ALGORITHM = _SCATTER_ALGORITHM; + static const cudaSharedMemConfig SMEM_CONFIG = _SMEM_CONFIG; + + typedef BlockRadixSortDownsweepTilesPolicy< + BLOCK_THREADS, + ITEMS_PER_THREAD, + LOAD_ALGORITHM, + LOAD_MODIFIER, + EXCHANGE_TIME_SLICING, + MEMOIZE_OUTER_SCAN, + INNER_SCAN_ALGORITHM, + SCATTER_ALGORITHM, + SMEM_CONFIG, + CUB_MAX(1, RADIX_BITS - 1)> AltPolicy; +}; + + +/****************************************************************************** + * Thread block abstractions + ******************************************************************************/ + +/** + * CTA-wide "downsweep" abstraction for distributing keys from + * a range of input tiles. + */ +template < + typename BlockRadixSortDownsweepTilesPolicy, + typename Key, + typename Value, + typename SizeT> +struct BlockRadixSortDownsweepTiles +{ + //--------------------------------------------------------------------- + // Type definitions and constants + //--------------------------------------------------------------------- + + // Appropriate unsigned-bits representation of Key + typedef typename Traits::UnsignedBits UnsignedBits; + + static const UnsignedBits MIN_KEY = Traits::MIN_KEY; + static const UnsignedBits MAX_KEY = Traits::MAX_KEY; + + static const BlockLoadAlgorithm LOAD_ALGORITHM = BlockRadixSortDownsweepTilesPolicy::LOAD_ALGORITHM; + static const PtxLoadModifier LOAD_MODIFIER = BlockRadixSortDownsweepTilesPolicy::LOAD_MODIFIER; + static const BlockScanAlgorithm INNER_SCAN_ALGORITHM = BlockRadixSortDownsweepTilesPolicy::INNER_SCAN_ALGORITHM; + static const RadixSortScatterAlgorithm SCATTER_ALGORITHM = BlockRadixSortDownsweepTilesPolicy::SCATTER_ALGORITHM; + static const cudaSharedMemConfig SMEM_CONFIG = BlockRadixSortDownsweepTilesPolicy::SMEM_CONFIG; + + enum + { + BLOCK_THREADS = BlockRadixSortDownsweepTilesPolicy::BLOCK_THREADS, + ITEMS_PER_THREAD = BlockRadixSortDownsweepTilesPolicy::ITEMS_PER_THREAD, + EXCHANGE_TIME_SLICING = BlockRadixSortDownsweepTilesPolicy::EXCHANGE_TIME_SLICING, + RADIX_BITS = BlockRadixSortDownsweepTilesPolicy::RADIX_BITS, + MEMOIZE_OUTER_SCAN = BlockRadixSortDownsweepTilesPolicy::MEMOIZE_OUTER_SCAN, + TILE_ITEMS = BLOCK_THREADS * ITEMS_PER_THREAD, + + RADIX_DIGITS = 1 << RADIX_BITS, + KEYS_ONLY = Equals::VALUE, + + WARP_THREADS = PtxArchProps::LOG_WARP_THREADS, + WARPS = (BLOCK_THREADS + WARP_THREADS - 1) / WARP_THREADS, + + BYTES_PER_SIZET = sizeof(SizeT), + LOG_BYTES_PER_SIZET = Log2::VALUE, + + LOG_SMEM_BANKS = PtxArchProps::LOG_SMEM_BANKS, + SMEM_BANKS = 1 << LOG_SMEM_BANKS, + + DIGITS_PER_SCATTER_PASS = BLOCK_THREADS / SMEM_BANKS, + SCATTER_PASSES = RADIX_DIGITS / DIGITS_PER_SCATTER_PASS, + + LOG_STORE_TXN_THREADS = LOG_SMEM_BANKS, + STORE_TXN_THREADS = 1 << LOG_STORE_TXN_THREADS, + }; + + // BlockRadixRank type + typedef BlockRadixRank< + BLOCK_THREADS, + RADIX_BITS, + MEMOIZE_OUTER_SCAN, + INNER_SCAN_ALGORITHM, + SMEM_CONFIG> BlockRadixRank; + + // BlockLoad type (keys) + typedef BlockLoad< + UnsignedBits*, + BLOCK_THREADS, + ITEMS_PER_THREAD, + LOAD_ALGORITHM, + LOAD_MODIFIER, + EXCHANGE_TIME_SLICING> BlockLoadKeys; + + // BlockLoad type (values) + typedef BlockLoad< + Value*, + BLOCK_THREADS, + ITEMS_PER_THREAD, + LOAD_ALGORITHM, + LOAD_MODIFIER, + EXCHANGE_TIME_SLICING> BlockLoadValues; + + // BlockExchange type (keys) + typedef BlockExchange< + UnsignedBits, + BLOCK_THREADS, + ITEMS_PER_THREAD, + EXCHANGE_TIME_SLICING> BlockExchangeKeys; + + // BlockExchange type (values) + typedef BlockExchange< + Value, + BLOCK_THREADS, + ITEMS_PER_THREAD, + EXCHANGE_TIME_SLICING> BlockExchangeValues; + + + /** + * Shared memory storage layout + */ + struct _TempStorage + { + SizeT relative_bin_offsets[RADIX_DIGITS + 1]; + bool short_circuit; + + union + { + typename BlockRadixRank::TempStorage ranking; + typename BlockLoadKeys::TempStorage load_keys; + typename BlockLoadValues::TempStorage load_values; + typename BlockExchangeKeys::TempStorage exchange_keys; + typename BlockExchangeValues::TempStorage exchange_values; + }; + }; + + + /// Alias wrapper allowing storage to be unioned + struct TempStorage : Uninitialized<_TempStorage> {}; + + + //--------------------------------------------------------------------- + // Thread fields + //--------------------------------------------------------------------- + + // Shared storage for this CTA + _TempStorage &temp_storage; + + // Input and output device pointers + UnsignedBits *d_keys_in; + UnsignedBits *d_keys_out; + Value *d_values_in; + Value *d_values_out; + + // The global scatter base offset for each digit (valid in the first RADIX_DIGITS threads) + SizeT bin_offset; + + // The least-significant bit position of the current digit to extract + int current_bit; + + // Whether to short-ciruit + bool short_circuit; + + + + //--------------------------------------------------------------------- + // Utility methods + //--------------------------------------------------------------------- + + /** + * Decodes given keys to lookup digit offsets in shared memory + */ + __device__ __forceinline__ void DecodeRelativeBinOffsets( + UnsignedBits (&twiddled_keys)[ITEMS_PER_THREAD], + SizeT (&relative_bin_offsets)[ITEMS_PER_THREAD]) + { + #pragma unroll + for (int KEY = 0; KEY < ITEMS_PER_THREAD; KEY++) + { + UnsignedBits digit = BFE(twiddled_keys[KEY], current_bit, RADIX_BITS); + + // Lookup base digit offset from shared memory + relative_bin_offsets[KEY] = temp_storage.relative_bin_offsets[digit]; + } + } + + + /** + * Scatter ranked items to global memory + */ + template + __device__ __forceinline__ void ScatterItems( + T (&items)[ITEMS_PER_THREAD], + int (&local_ranks)[ITEMS_PER_THREAD], + SizeT (&relative_bin_offsets)[ITEMS_PER_THREAD], + T *d_out, + SizeT valid_items) + { + for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ++ITEM) + { + // Scatter if not out-of-bounds + if (FULL_TILE || (local_ranks[ITEM] < valid_items)) + { + d_out[relative_bin_offsets[ITEM] + local_ranks[ITEM]] = items[ITEM]; + } + } + } + + + /** + * Scatter ranked keys directly to global memory + */ + template + __device__ __forceinline__ void ScatterKeys( + UnsignedBits (&twiddled_keys)[ITEMS_PER_THREAD], + SizeT (&relative_bin_offsets)[ITEMS_PER_THREAD], + int (&ranks)[ITEMS_PER_THREAD], + SizeT valid_items, + Int2Type scatter_algorithm) + { + // Compute scatter offsets + DecodeRelativeBinOffsets(twiddled_keys, relative_bin_offsets); + + // Untwiddle keys before outputting + UnsignedBits keys[ITEMS_PER_THREAD]; + + #pragma unroll + for (int KEY = 0; KEY < ITEMS_PER_THREAD; KEY++) + { + keys[KEY] = Traits::TwiddleOut(twiddled_keys[KEY]); + } + + // Scatter to global + ScatterItems(keys, ranks, relative_bin_offsets, d_keys_out, valid_items); + } + + + /** + * Scatter ranked keys through shared memory, then to global memory + */ + template + __device__ __forceinline__ void ScatterKeys( + UnsignedBits (&twiddled_keys)[ITEMS_PER_THREAD], + SizeT (&relative_bin_offsets)[ITEMS_PER_THREAD], + int (&ranks)[ITEMS_PER_THREAD], + SizeT valid_items, + Int2Type scatter_algorithm) + { + // Exchange keys through shared memory + BlockExchangeKeys(temp_storage.exchange_keys).ScatterToStriped(twiddled_keys, ranks); + + // Compute striped local ranks + int local_ranks[ITEMS_PER_THREAD]; + for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ++ITEM) + { + local_ranks[ITEM] = threadIdx.x + (ITEM * BLOCK_THREADS); + } + + // Scatter directly + ScatterKeys( + twiddled_keys, + relative_bin_offsets, + local_ranks, + valid_items, + Int2Type()); + } + + + /** + * Scatter ranked values directly to global memory + */ + template + __device__ __forceinline__ void ScatterValues( + Value (&values)[ITEMS_PER_THREAD], + SizeT (&relative_bin_offsets)[ITEMS_PER_THREAD], + int (&ranks)[ITEMS_PER_THREAD], + SizeT valid_items, + Int2Type scatter_algorithm) + { + // Scatter to global + ScatterItems(values, ranks, relative_bin_offsets, d_values_out, valid_items); + } + + + /** + * Scatter ranked values through shared memory, then to global memory + */ + template + __device__ __forceinline__ void ScatterValues( + Value (&values)[ITEMS_PER_THREAD], + SizeT (&relative_bin_offsets)[ITEMS_PER_THREAD], + int (&ranks)[ITEMS_PER_THREAD], + SizeT valid_items, + Int2Type scatter_algorithm) + { + __syncthreads(); + + // Exchange keys through shared memory + BlockExchangeValues(temp_storage.exchange_values).ScatterToStriped(values, ranks); + + // Compute striped local ranks + int local_ranks[ITEMS_PER_THREAD]; + for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ++ITEM) + { + local_ranks[ITEM] = threadIdx.x + (ITEM * BLOCK_THREADS); + } + + // Scatter directly + ScatterValues( + values, + relative_bin_offsets, + local_ranks, + valid_items, + Int2Type()); + } + + + /** + * Load a tile of items (specialized for full tile) + */ + template + __device__ __forceinline__ void LoadItems( + BlockLoadT &block_loader, + T (&items)[ITEMS_PER_THREAD], + T *d_in, + SizeT valid_items, + Int2Type is_full_tile) + { + block_loader.Load(d_in, items); + } + + + /** + * Load a tile of items (specialized for partial tile) + */ + template + __device__ __forceinline__ void LoadItems( + BlockLoadT &block_loader, + T (&items)[ITEMS_PER_THREAD], + T *d_in, + SizeT valid_items, + Int2Type is_full_tile) + { + block_loader.Load(d_in, items, valid_items); + } + + + /** + * Truck along associated values + */ + template + __device__ __forceinline__ void GatherScatterValues( + _Value (&values)[ITEMS_PER_THREAD], + SizeT (&relative_bin_offsets)[ITEMS_PER_THREAD], + int (&ranks)[ITEMS_PER_THREAD], + SizeT block_offset, + SizeT valid_items) + { + BlockLoadValues loader(temp_storage.load_values); + LoadItems( + loader, + values, + d_values_in + block_offset, + valid_items, + Int2Type()); + + ScatterValues( + values, + relative_bin_offsets, + ranks, + valid_items, + Int2Type()); + } + + + /** + * Truck along associated values (specialized for key-only sorting) + */ + template + __device__ __forceinline__ void GatherScatterValues( + NullType (&values)[ITEMS_PER_THREAD], + SizeT (&relative_bin_offsets)[ITEMS_PER_THREAD], + int (&ranks)[ITEMS_PER_THREAD], + SizeT block_offset, + SizeT valid_items) + {} + + + /** + * Process tile + */ + template + __device__ __forceinline__ void ProcessTile( + SizeT block_offset, + const SizeT &valid_items = TILE_ITEMS) + { + // Per-thread tile data + UnsignedBits keys[ITEMS_PER_THREAD]; // Keys + UnsignedBits twiddled_keys[ITEMS_PER_THREAD]; // Twiddled keys + int ranks[ITEMS_PER_THREAD]; // For each key, the local rank within the CTA + SizeT relative_bin_offsets[ITEMS_PER_THREAD]; // For each key, the global scatter base offset of the corresponding digit + + if (LOAD_ALGORITHM != BLOCK_LOAD_DIRECT) __syncthreads(); + + // Assign max-key to all keys + #pragma unroll + for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ++ITEM) + { + keys[ITEM] = MAX_KEY; + } + + // Load tile of keys + BlockLoadKeys loader(temp_storage.load_keys); + LoadItems( + loader, + keys, + d_keys_in + block_offset, + valid_items, + Int2Type()); + + __syncthreads(); + + // Twiddle key bits if necessary + #pragma unroll + for (int KEY = 0; KEY < ITEMS_PER_THREAD; KEY++) + { + twiddled_keys[KEY] = Traits::TwiddleIn(keys[KEY]); + } + + // Rank the twiddled keys + int inclusive_digit_prefix; + BlockRadixRank(temp_storage.ranking).RankKeys( + twiddled_keys, + ranks, + current_bit, + inclusive_digit_prefix); + + // Update global scatter base offsets for each digit + if ((BLOCK_THREADS == RADIX_DIGITS) || (threadIdx.x < RADIX_DIGITS)) + { + int exclusive_digit_prefix; + + // Get exclusive digit prefix from inclusive prefix +#if CUB_PTX_ARCH >= 300 + exclusive_digit_prefix = ShuffleUp(inclusive_digit_prefix, 1); + if (threadIdx.x == 0) + exclusive_digit_prefix = 0; +#else + volatile int* exchange = reinterpret_cast(temp_storage.relative_bin_offsets); + exchange[threadIdx.x] = 0; + exchange[threadIdx.x + 1] = inclusive_digit_prefix; + exclusive_digit_prefix = exchange[threadIdx.x]; +#endif + + bin_offset -= exclusive_digit_prefix; + temp_storage.relative_bin_offsets[threadIdx.x] = bin_offset; + bin_offset += inclusive_digit_prefix; + } + + __syncthreads(); + + // Scatter keys + ScatterKeys(twiddled_keys, relative_bin_offsets, ranks, valid_items, Int2Type()); + + // Gather/scatter values + Value values[ITEMS_PER_THREAD]; + GatherScatterValues(values, relative_bin_offsets, ranks, block_offset, valid_items); + } + + + /** + * Copy tiles within the range of input + */ + template + __device__ __forceinline__ void Copy( + T *d_in, + T *d_out, + SizeT block_offset, + SizeT block_oob) + { + // Simply copy the input + while (block_offset + TILE_ITEMS <= block_oob) + { + T items[ITEMS_PER_THREAD]; + + LoadStriped(threadIdx.x, d_in + block_offset, items); + __syncthreads(); + StoreStriped(threadIdx.x, d_out + block_offset, items); + + block_offset += TILE_ITEMS; + } + + // Clean up last partial tile with guarded-I/O + if (block_offset < block_oob) + { + SizeT valid_items = block_oob - block_offset; + + T items[ITEMS_PER_THREAD]; + + LoadStriped(threadIdx.x, d_in + block_offset, items, valid_items); + __syncthreads(); + StoreStriped(threadIdx.x, d_out + block_offset, items, valid_items); + } + } + + + /** + * Copy tiles within the range of input (specialized for NullType) + */ + __device__ __forceinline__ void Copy( + NullType *d_in, + NullType *d_out, + SizeT block_offset, + SizeT block_oob) + {} + + + //--------------------------------------------------------------------- + // Interface + //--------------------------------------------------------------------- + + /** + * Constructor + */ + __device__ __forceinline__ BlockRadixSortDownsweepTiles( + TempStorage &temp_storage, + SizeT bin_offset, + Key *d_keys_in, + Key *d_keys_out, + Value *d_values_in, + Value *d_values_out, + int current_bit) + : + temp_storage(temp_storage.Alias()), + bin_offset(bin_offset), + d_keys_in(reinterpret_cast(d_keys_in)), + d_keys_out(reinterpret_cast(d_keys_out)), + d_values_in(d_values_in), + d_values_out(d_values_out), + current_bit(current_bit), + short_circuit(false) + {} + + + /** + * Constructor + */ + __device__ __forceinline__ BlockRadixSortDownsweepTiles( + TempStorage &temp_storage, + SizeT num_items, + SizeT *d_spine, + Key *d_keys_in, + Key *d_keys_out, + Value *d_values_in, + Value *d_values_out, + int current_bit) + : + temp_storage(temp_storage.Alias()), + d_keys_in(reinterpret_cast(d_keys_in)), + d_keys_out(reinterpret_cast(d_keys_out)), + d_values_in(d_values_in), + d_values_out(d_values_out), + current_bit(current_bit) + { + // Load digit bin offsets (each of the first RADIX_DIGITS threads will load an offset for that digit) + if (threadIdx.x < RADIX_DIGITS) + { + // Short circuit if the first block's histogram has only bin counts of only zeros or problem-size + SizeT first_block_bin_offset = d_spine[gridDim.x * threadIdx.x]; + int predicate = ((first_block_bin_offset == 0) || (first_block_bin_offset == num_items)); + this->temp_storage.short_circuit = WarpAll(predicate); + + // Load my block's bin offset for my bin + bin_offset = d_spine[(gridDim.x * threadIdx.x) + blockIdx.x]; + } + + __syncthreads(); + + short_circuit = this->temp_storage.short_circuit; + } + + + /** + * Distribute keys from a segment of input tiles. + */ + __device__ __forceinline__ void ProcessTiles( + SizeT block_offset, + const SizeT &block_oob) + { + if (short_circuit) + { + // Copy keys + Copy(d_keys_in, d_keys_out, block_offset, block_oob); + + // Copy values + Copy(d_values_in, d_values_out, block_offset, block_oob); + } + else + { + // Process full tiles of tile_items + while (block_offset + TILE_ITEMS <= block_oob) + { + ProcessTile(block_offset); + block_offset += TILE_ITEMS; + } + + // Clean up last partial tile with guarded-I/O + if (block_offset < block_oob) + { + ProcessTile(block_offset, block_oob - block_offset); + } + } + } +}; + + + +} // CUB namespace +CUB_NS_POSTFIX // Optional outer namespace(s) + diff --git a/lib/kokkos/TPL/cub/device/block/block_radix_sort_upsweep_tiles.cuh b/lib/kokkos/TPL/cub/device/block/block_radix_sort_upsweep_tiles.cuh new file mode 100755 index 0000000000..22f8c9c75e --- /dev/null +++ b/lib/kokkos/TPL/cub/device/block/block_radix_sort_upsweep_tiles.cuh @@ -0,0 +1,464 @@ +/****************************************************************************** + * Copyright (c) 2011, Duane Merrill. All rights reserved. + * Copyright (c) 2011-2013, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the NVIDIA CORPORATION nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + ******************************************************************************/ + +/** + * \file + * BlockRadixSortUpsweepTiles implements a stateful abstraction of CUDA thread blocks for participating in device-wide radix sort upsweep. + */ + +#pragma once + +#include "../../thread/thread_reduce.cuh" +#include "../../thread/thread_load.cuh" +#include "../../block/block_load.cuh" +#include "../../util_type.cuh" +#include "../../util_namespace.cuh" + +/// Optional outer namespace(s) +CUB_NS_PREFIX + +/// CUB namespace +namespace cub { + +/****************************************************************************** + * Tuning policy types + ******************************************************************************/ + +/** + * Tuning policy for BlockRadixSortUpsweepTiles + */ +template < + int _BLOCK_THREADS, ///< The number of threads per CTA + int _ITEMS_PER_THREAD, ///< The number of items to load per thread per tile + PtxLoadModifier _LOAD_MODIFIER, ///< Load cache-modifier + int _RADIX_BITS> ///< The number of radix bits, i.e., log2(bins) +struct BlockRadixSortUpsweepTilesPolicy +{ + enum + { + BLOCK_THREADS = _BLOCK_THREADS, + ITEMS_PER_THREAD = _ITEMS_PER_THREAD, + RADIX_BITS = _RADIX_BITS, + TILE_ITEMS = BLOCK_THREADS * ITEMS_PER_THREAD, + }; + + static const PtxLoadModifier LOAD_MODIFIER = _LOAD_MODIFIER; + + typedef BlockRadixSortUpsweepTilesPolicy< + BLOCK_THREADS, + ITEMS_PER_THREAD, + LOAD_MODIFIER, + CUB_MAX(1, RADIX_BITS - 1)> AltPolicy; +}; + + +/****************************************************************************** + * Thread block abstractions + ******************************************************************************/ + +/** + * \brief BlockRadixSortUpsweepTiles implements a stateful abstraction of CUDA thread blocks for participating in device-wide radix sort upsweep. + * + * Computes radix digit histograms over a range of input tiles. + */ +template < + typename BlockRadixSortUpsweepTilesPolicy, + typename Key, + typename SizeT> +struct BlockRadixSortUpsweepTiles +{ + + //--------------------------------------------------------------------- + // Type definitions and constants + //--------------------------------------------------------------------- + + typedef typename Traits::UnsignedBits UnsignedBits; + + // Integer type for digit counters (to be packed into words of PackedCounters) + typedef unsigned char DigitCounter; + + // Integer type for packing DigitCounters into columns of shared memory banks + typedef unsigned int PackedCounter; + + static const PtxLoadModifier LOAD_MODIFIER = BlockRadixSortUpsweepTilesPolicy::LOAD_MODIFIER; + + enum + { + RADIX_BITS = BlockRadixSortUpsweepTilesPolicy::RADIX_BITS, + BLOCK_THREADS = BlockRadixSortUpsweepTilesPolicy::BLOCK_THREADS, + KEYS_PER_THREAD = BlockRadixSortUpsweepTilesPolicy::ITEMS_PER_THREAD, + + RADIX_DIGITS = 1 << RADIX_BITS, + + LOG_WARP_THREADS = PtxArchProps::LOG_WARP_THREADS, + WARP_THREADS = 1 << LOG_WARP_THREADS, + WARPS = (BLOCK_THREADS + WARP_THREADS - 1) / WARP_THREADS, + + TILE_ITEMS = BLOCK_THREADS * KEYS_PER_THREAD, + + BYTES_PER_COUNTER = sizeof(DigitCounter), + LOG_BYTES_PER_COUNTER = Log2::VALUE, + + PACKING_RATIO = sizeof(PackedCounter) / sizeof(DigitCounter), + LOG_PACKING_RATIO = Log2::VALUE, + + LOG_COUNTER_LANES = CUB_MAX(0, RADIX_BITS - LOG_PACKING_RATIO), + COUNTER_LANES = 1 << LOG_COUNTER_LANES, + + // To prevent counter overflow, we must periodically unpack and aggregate the + // digit counters back into registers. Each counter lane is assigned to a + // warp for aggregation. + + LANES_PER_WARP = CUB_MAX(1, (COUNTER_LANES + WARPS - 1) / WARPS), + + // Unroll tiles in batches without risk of counter overflow + UNROLL_COUNT = CUB_MIN(64, 255 / KEYS_PER_THREAD), + UNROLLED_ELEMENTS = UNROLL_COUNT * TILE_ITEMS, + }; + + + + /** + * Shared memory storage layout + */ + struct _TempStorage + { + union + { + DigitCounter digit_counters[COUNTER_LANES][BLOCK_THREADS][PACKING_RATIO]; + PackedCounter packed_counters[COUNTER_LANES][BLOCK_THREADS]; + SizeT digit_partials[RADIX_DIGITS][WARP_THREADS + 1]; + }; + }; + + + /// Alias wrapper allowing storage to be unioned + struct TempStorage : Uninitialized<_TempStorage> {}; + + + //--------------------------------------------------------------------- + // Thread fields (aggregate state bundle) + //--------------------------------------------------------------------- + + // Shared storage for this CTA + _TempStorage &temp_storage; + + // Thread-local counters for periodically aggregating composite-counter lanes + SizeT local_counts[LANES_PER_WARP][PACKING_RATIO]; + + // Input and output device pointers + UnsignedBits *d_keys_in; + + // The least-significant bit position of the current digit to extract + int current_bit; + + + + //--------------------------------------------------------------------- + // Helper structure for templated iteration + //--------------------------------------------------------------------- + + // Iterate + template + struct Iterate + { + enum { + HALF = (MAX / 2), + }; + + // BucketKeys + static __device__ __forceinline__ void BucketKeys( + BlockRadixSortUpsweepTiles &cta, + UnsignedBits keys[KEYS_PER_THREAD]) + { + cta.Bucket(keys[COUNT]); + + // Next + Iterate::BucketKeys(cta, keys); + } + + // ProcessTiles + static __device__ __forceinline__ void ProcessTiles(BlockRadixSortUpsweepTiles &cta, SizeT block_offset) + { + // Next + Iterate<1, HALF>::ProcessTiles(cta, block_offset); + Iterate<1, MAX - HALF>::ProcessTiles(cta, block_offset + (HALF * TILE_ITEMS)); + } + }; + + // Terminate + template + struct Iterate + { + // BucketKeys + static __device__ __forceinline__ void BucketKeys(BlockRadixSortUpsweepTiles &cta, UnsignedBits keys[KEYS_PER_THREAD]) {} + + // ProcessTiles + static __device__ __forceinline__ void ProcessTiles(BlockRadixSortUpsweepTiles &cta, SizeT block_offset) + { + cta.ProcessFullTile(block_offset); + } + }; + + + //--------------------------------------------------------------------- + // Utility methods + //--------------------------------------------------------------------- + + /** + * Decode a key and increment corresponding smem digit counter + */ + __device__ __forceinline__ void Bucket(UnsignedBits key) + { + // Perform transform op + UnsignedBits converted_key = Traits::TwiddleIn(key); + + // Add in sub-counter offset + UnsignedBits sub_counter = BFE(converted_key, current_bit, LOG_PACKING_RATIO); + + // Add in row offset + UnsignedBits row_offset = BFE(converted_key, current_bit + LOG_PACKING_RATIO, LOG_COUNTER_LANES); + + // Increment counter + temp_storage.digit_counters[row_offset][threadIdx.x][sub_counter]++; + + } + + + /** + * Reset composite counters + */ + __device__ __forceinline__ void ResetDigitCounters() + { + #pragma unroll + for (int LANE = 0; LANE < COUNTER_LANES; LANE++) + { + temp_storage.packed_counters[LANE][threadIdx.x] = 0; + } + } + + + /** + * Reset the unpacked counters in each thread + */ + __device__ __forceinline__ void ResetUnpackedCounters() + { + #pragma unroll + for (int LANE = 0; LANE < LANES_PER_WARP; LANE++) + { + #pragma unroll + for (int UNPACKED_COUNTER = 0; UNPACKED_COUNTER < PACKING_RATIO; UNPACKED_COUNTER++) + { + local_counts[LANE][UNPACKED_COUNTER] = 0; + } + } + } + + + /** + * Extracts and aggregates the digit counters for each counter lane + * owned by this warp + */ + __device__ __forceinline__ void UnpackDigitCounts() + { + unsigned int warp_id = threadIdx.x >> LOG_WARP_THREADS; + unsigned int warp_tid = threadIdx.x & (WARP_THREADS - 1); + + #pragma unroll + for (int LANE = 0; LANE < LANES_PER_WARP; LANE++) + { + const int counter_lane = (LANE * WARPS) + warp_id; + if (counter_lane < COUNTER_LANES) + { + #pragma unroll + for (int PACKED_COUNTER = 0; PACKED_COUNTER < BLOCK_THREADS; PACKED_COUNTER += WARP_THREADS) + { + #pragma unroll + for (int UNPACKED_COUNTER = 0; UNPACKED_COUNTER < PACKING_RATIO; UNPACKED_COUNTER++) + { + SizeT counter = temp_storage.digit_counters[counter_lane][warp_tid + PACKED_COUNTER][UNPACKED_COUNTER]; + local_counts[LANE][UNPACKED_COUNTER] += counter; + } + } + } + } + } + + + /** + * Places unpacked counters into smem for final digit reduction + */ + __device__ __forceinline__ void ReduceUnpackedCounts(SizeT &bin_count) + { + unsigned int warp_id = threadIdx.x >> LOG_WARP_THREADS; + unsigned int warp_tid = threadIdx.x & (WARP_THREADS - 1); + + // Place unpacked digit counters in shared memory + #pragma unroll + for (int LANE = 0; LANE < LANES_PER_WARP; LANE++) + { + int counter_lane = (LANE * WARPS) + warp_id; + if (counter_lane < COUNTER_LANES) + { + int digit_row = counter_lane << LOG_PACKING_RATIO; + + #pragma unroll + for (int UNPACKED_COUNTER = 0; UNPACKED_COUNTER < PACKING_RATIO; UNPACKED_COUNTER++) + { + temp_storage.digit_partials[digit_row + UNPACKED_COUNTER][warp_tid] = + local_counts[LANE][UNPACKED_COUNTER]; + } + } + } + + __syncthreads(); + + // Rake-reduce bin_count reductions + if (threadIdx.x < RADIX_DIGITS) + { + bin_count = ThreadReduce( + temp_storage.digit_partials[threadIdx.x], + Sum()); + } + } + + + /** + * Processes a single, full tile + */ + __device__ __forceinline__ void ProcessFullTile(SizeT block_offset) + { + // Tile of keys + UnsignedBits keys[KEYS_PER_THREAD]; + + LoadStriped(threadIdx.x, d_keys_in + block_offset, keys); + + // Prevent hoisting +// __threadfence_block(); +// __syncthreads(); + + // Bucket tile of keys + Iterate<0, KEYS_PER_THREAD>::BucketKeys(*this, keys); + } + + + /** + * Processes a single load (may have some threads masked off) + */ + __device__ __forceinline__ void ProcessPartialTile( + SizeT block_offset, + const SizeT &block_oob) + { + // Process partial tile if necessary using single loads + block_offset += threadIdx.x; + while (block_offset < block_oob) + { + // Load and bucket key + UnsignedBits key = ThreadLoad(d_keys_in + block_offset); + Bucket(key); + block_offset += BLOCK_THREADS; + } + } + + + //--------------------------------------------------------------------- + // Interface + //--------------------------------------------------------------------- + + /** + * Constructor + */ + __device__ __forceinline__ BlockRadixSortUpsweepTiles( + TempStorage &temp_storage, + Key *d_keys_in, + int current_bit) + : + temp_storage(temp_storage.Alias()), + d_keys_in(reinterpret_cast(d_keys_in)), + current_bit(current_bit) + {} + + + /** + * Compute radix digit histograms from a segment of input tiles. + */ + __device__ __forceinline__ void ProcessTiles( + SizeT block_offset, + const SizeT &block_oob, + SizeT &bin_count) ///< [out] The digit count for tid'th bin (output param, valid in the first RADIX_DIGITS threads) + { + // Reset digit counters in smem and unpacked counters in registers + ResetDigitCounters(); + ResetUnpackedCounters(); + + // Unroll batches of full tiles + while (block_offset + UNROLLED_ELEMENTS <= block_oob) + { + Iterate<0, UNROLL_COUNT>::ProcessTiles(*this, block_offset); + block_offset += UNROLLED_ELEMENTS; + + __syncthreads(); + + // Aggregate back into local_count registers to prevent overflow + UnpackDigitCounts(); + + __syncthreads(); + + // Reset composite counters in lanes + ResetDigitCounters(); + } + + // Unroll single full tiles + while (block_offset + TILE_ITEMS <= block_oob) + { + ProcessFullTile(block_offset); + block_offset += TILE_ITEMS; + } + + // Process partial tile if necessary + ProcessPartialTile( + block_offset, + block_oob); + + __syncthreads(); + + // Aggregate back into local_count registers + UnpackDigitCounts(); + + __syncthreads(); + + // Final raking reduction of counts by bin + ReduceUnpackedCounts(bin_count); + } + +}; + + +} // CUB namespace +CUB_NS_POSTFIX // Optional outer namespace(s) + diff --git a/lib/kokkos/TPL/cub/device/block/block_reduce_by_key_tiles.cuh b/lib/kokkos/TPL/cub/device/block/block_reduce_by_key_tiles.cuh new file mode 100755 index 0000000000..99e1980b6f --- /dev/null +++ b/lib/kokkos/TPL/cub/device/block/block_reduce_by_key_tiles.cuh @@ -0,0 +1,399 @@ +/****************************************************************************** + * Copyright (c) 2011, Duane Merrill. All rights reserved. + * Copyright (c) 2011-2013, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the NVIDIA CORPORATION nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + ******************************************************************************/ + +/** + * \file + * cub::BlockReduceByKeyiles implements a stateful abstraction of CUDA thread blocks for participating in device-wide reduce-value-by-key. + */ + +#pragma once + +#include + +#include "scan_tiles_types.cuh" +#include "../../block/block_load.cuh" +#include "../../block/block_discontinuity.cuh" +#include "../../block/block_scan.cuh" +#include "../../util_namespace.cuh" + +/// Optional outer namespace(s) +CUB_NS_PREFIX + +/// CUB namespace +namespace cub { + + +/****************************************************************************** + * Utility data types + ******************************************************************************/ + +/// Scan tuple data type for reduce-value-by-key +template +struct ReduceByKeyuple +{ + Value value; // Initially set as value, contains segment aggregate after prefix scan + SizeT flag; // Initially set as a tail flag, contains scatter offset after prefix scan +}; + + +/// Binary reduce-by-key scan operator +template +struct ReduceByKeyScanOp +{ + /// Reduction functor + ReductionOp reduction_op; + + /// Constructor + ReduceByKeyScanOp(ReductionOp reduction_op) : reduction_op(reduction_op) + {} + + /// Binary scan operator + template + __device__ __forceinline__ ReduceByKeyuple operator()( + const ReduceByKeyuple &first, + const ReduceByKeyuple &second) + { + ReduceByKeyuple retval; + retval.val = (second.flag) ? second.val : reduction_op(first.val, second.val); + retval.flag = first.flag + second.flag; + return retval; + } +}; + + + +/****************************************************************************** + * Tuning policy types + ******************************************************************************/ + +/** + * Tuning policy for BlockReduceByKeyiles + */ +template < + int _BLOCK_THREADS, + int _ITEMS_PER_THREAD, + BlockLoadAlgorithm _LOAD_ALGORITHM, + bool _LOAD_WARP_TIME_SLICING, + PtxLoadModifier _LOAD_MODIFIER, + BlockScanAlgorithm _SCAN_ALGORITHM> +struct BlockReduceByKeyilesPolicy +{ + enum + { + BLOCK_THREADS = _BLOCK_THREADS, + ITEMS_PER_THREAD = _ITEMS_PER_THREAD, + LOAD_WARP_TIME_SLICING = _LOAD_WARP_TIME_SLICING, + }; + + static const BlockLoadAlgorithm LOAD_ALGORITHM = _LOAD_ALGORITHM; + static const PtxLoadModifier LOAD_MODIFIER = _LOAD_MODIFIER; + static const BlockScanAlgorithm SCAN_ALGORITHM = _SCAN_ALGORITHM; +}; + + +/****************************************************************************** + * Thread block abstractions + ******************************************************************************/ + +/** + * \brief BlockReduceByKeyiles implements a stateful abstraction of CUDA thread blocks for participating in device-wide prefix scan. + */ +template < + typename BlockReduceByKeyilesPolicy, ///< Tuning policy + typename KeyInputIteratorRA, ///< Random-access input iterator type for keys + typename KeyOutputIteratorRA, ///< Random-access output iterator type for keys + typename ValueInputIteratorRA, ///< Random-access input iterator type for values + typename ValueOutputIteratorRA, ///< Random-access output iterator type for values + typename ReductionOp, ///< Reduction functor type + typename SizeT> ///< Offset integer type +struct BlockReduceByKeyiles +{ + //--------------------------------------------------------------------- + // Types and constants + //--------------------------------------------------------------------- + + // Data types of input iterators + typedef typename std::iterator_traits::value_type Key; // Key data type + typedef typename std::iterator_traits::value_type Value; // Value data type + + // Constants + enum + { + BLOCK_THREADS = BlockReduceByKeyilesPolicy::BLOCK_THREADS, + ITEMS_PER_THREAD = BlockReduceByKeyilesPolicy::ITEMS_PER_THREAD, + TILE_ITEMS = BLOCK_THREADS * ITEMS_PER_THREAD, + STATUS_PADDING = PtxArchProps::WARP_THREADS, + }; + + // Block load type for keys + typedef BlockLoad< + KeyInputIteratorRA, + BlockReduceByKeyilesPolicy::BLOCK_THREADS, + BlockReduceByKeyilesPolicy::ITEMS_PER_THREAD, + BlockReduceByKeyilesPolicy::LOAD_ALGORITHM, + BlockReduceByKeyilesPolicy::LOAD_MODIFIER, + BlockReduceByKeyilesPolicy::LOAD_WARP_TIME_SLICING> BlockLoadKeys; + + // Block load type for values + typedef BlockLoad< + ValueInputIteratorRA, + BlockReduceByKeyilesPolicy::BLOCK_THREADS, + BlockReduceByKeyilesPolicy::ITEMS_PER_THREAD, + BlockReduceByKeyilesPolicy::LOAD_ALGORITHM, + BlockReduceByKeyilesPolicy::LOAD_MODIFIER, + BlockReduceByKeyilesPolicy::LOAD_WARP_TIME_SLICING> BlockLoadValues; + + // Block discontinuity type for setting tail flags + typedef BlockDiscontinuity BlockDiscontinuityKeys; + + // Scan tuple type + typedef ReduceByKeyuple ScanTuple; + + // Tile status descriptor type + typedef ScanTileDescriptor ScanTileDescriptorT; + + // Block scan functor type + typedef ReduceByKeyScanOp ScanOp; + + // Block scan prefix callback type + typedef DeviceScanBlockPrefixOp PrefixCallback; + + // Block scan type + typedef BlockScan< + ScanTuple, + BlockReduceByKeyilesPolicy::BLOCK_THREADS, + BlockReduceByKeyilesPolicy::SCAN_ALGORITHM> BlockScanT; + + /// Shared memory type for this threadblock + struct _TempStorage + { + union + { + typename BlockLoadKeys::TempStorage load_keys; // Smem needed for loading tiles of keys + typename BlockLoadValues::TempStorage load_values; // Smem needed for loading tiles of values + struct + { + typename BlockScanT::TempStorage scan; // Smem needed for tile scanning + typename PrefixCallback::TempStorage prefix; // Smem needed for cooperative prefix callback + }; + }; + + typename BlockDiscontinuityKeys::TempStorage flagging; // Smem needed for tile scanning + SizeT tile_idx; // Shared tile index + }; + + /// Alias wrapper allowing storage to be unioned + struct TempStorage : Uninitialized<_TempStorage> {}; + + + + //--------------------------------------------------------------------- + // Per-thread fields + //--------------------------------------------------------------------- + + _TempStorage &temp_storage; ///< Reference to temp_storage + KeyInputIteratorRA d_keys_in; ///< Key input data + KeyOutputIteratorRA d_keys_out; ///< Key output data + ValueInputIteratorRA d_values_in; ///< Value input data + ValueOutputIteratorRA d_values_out; ///< Value output data + ScanTileDescriptorT *d_tile_status; ///< Global list of tile status + ScanOp scan_op; ///< Binary scan operator + int num_tiles; ///< Total number of input tiles for the entire problem + SizeT num_items; ///< Total number of scan items for the entire problem + + + //--------------------------------------------------------------------- + // Interface + //--------------------------------------------------------------------- + + // Constructor + __device__ __forceinline__ + BlockReduceByKeyiles( + TempStorage &temp_storage, ///< Reference to temp_storage + KeyInputIteratorRA d_keys_in, ///< Key input data + KeyOutputIteratorRA d_keys_out, ///< Key output data + ValueInputIteratorRA d_values_in, ///< Value input data + ValueOutputIteratorRA d_values_out, ///< Value output data + ScanTileDescriptorT *d_tile_status, ///< Global list of tile status + ReductionOp reduction_op, ///< Binary scan operator + int num_tiles, ///< Total number of input tiles for the entire problem + SizeT num_items) ///< Total number of scan items for the entire problem + : + temp_storage(temp_storage.Alias()), + d_keys_in(d_keys_in), + d_keys_out(d_keys_out), + d_values_in(d_values_in), + d_values_out(d_values_out), + d_tile_status(d_tile_status), + scan_op(reduction_op), + num_tiles(num_tiles), + num_items(num_items) + {} + + + /** + * Process a tile of input + */ + template + __device__ __forceinline__ void ConsumeTile( + int tile_idx, ///< Tile index + SizeT block_offset, ///< Tile offset + int valid_items = TILE_ITEMS) ///< Number of valid items in the tile + { + Key keys[ITEMS_PER_THREAD]; + Value values[ITEMS_PER_THREAD]; + int tail_flags[ITEMS_PER_THREAD]; + ScanTuple scan_tuples[ITEMS_PER_THREAD]; + + // Load keys + if (FULL_TILE) + BlockLoadKeys(temp_storage.load_keys).Load(d_keys_in + block_offset, keys); + else + BlockLoadKeys(temp_storage.load_keys).Load(d_keys_in + block_offset, keys, valid_items); + + // Set tail flags + if (tile_idx == num_tiles - 1) + { + // Last tile + BlockDiscontinuityKeys(temp_storage.flagging).FlagTails(tail_flags, keys, Equality()); + } + else + { + // Preceding tiles require the first element of the next tile + Key tile_suffix_item; + if (threadIdx.x == 0) + tile_suffix_item = d_keys_in[block_offset + TILE_ITEMS]; + + BlockDiscontinuityKeys(temp_storage.flagging).FlagTails(tail_flags, keys, Equality(), tile_suffix_item); + } + + __syncthreads(); + + // Load values + if (FULL_TILE) + BlockLoadValues(temp_storage.load_values).Load(d_values_in + block_offset, values); + else + BlockLoadValues(temp_storage.load_values).Load(d_values_in + block_offset, values, valid_items); + + // Assemble scan tuples + #pragma unroll + for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ++ITEM) + { + scan_tuples[ITEM].value = values[ITEM]; + scan_tuples[ITEM].flag = tail_flags[ITEM]; + } + + __syncthreads(); + + // Perform inclusive prefix scan + ScanTuple block_aggregate; + if (tile_idx == 0) + { + // Without prefix callback + BlockScanT(temp_storage.scan).InclusiveScan(scan_tuples, scan_tuples, scan_op, block_aggregate); + + // Update tile status + if (threadIdx.x == 0) + ScanTileDescriptorT::SetPrefix(d_tile_status, block_aggregate); + } + else + { + // With prefix callback + PrefixCallback prefix_op(d_tile_status, temp_storage.prefix, scan_op, tile_idx); + BlockScanT(temp_storage.scan).InclusiveScan(scan_tuples, scan_tuples, scan_op, block_aggregate, prefix_op); + } + + // Scatter flagged keys and values to output + #pragma unroll + for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ++ITEM) + { + int tile_item = (threadIdx.x * ITEMS_PER_THREAD) + ITEM; + + // Set the head flag on the last item in a partially-full tile + if (!FULL_TILE && (tile_item == valid_items - 1)) + tail_flags[ITEM] = 1; + + // Decrement scatter offset + scan_tuples[ITEM].flag--; + + // Scatter key and aggregate value if flagged and in range + if ((FULL_TILE || (tile_item < valid_items)) && (tail_flags[ITEM])) + { + d_keys_out[scan_tuples[ITEM].flag] = keys[ITEM]; + d_values_out[scan_tuples[ITEM].flag] = scan_tuples[ITEM].value; + } + } + } + + + + /** + * Dequeue and scan tiles of elements + */ + __device__ __forceinline__ void ProcessTiles(GridQueue queue) ///< Queue descriptor for assigning tiles of work to thread blocks + { + // We give each thread block at least one tile of input + int tile_idx = blockIdx.x; + + // Consume full tiles of input + SizeT block_offset = SizeT(TILE_ITEMS) * tile_idx; + while (block_offset + TILE_ITEMS <= num_items) + { + ConsumeTile(tile_idx, block_offset); + + // Get next tile +#if CUB_PTX_ARCH < 200 + // No concurrent kernels allowed, so just stripe tiles + tile_idx += gridDim.x; +#else + // Concurrent kernels are allowed, so we must only use active blocks to dequeue tile indices + if (threadIdx.x == 0) + temp_storage.tile_idx = queue.Drain(1) + gridDim.x; + + __syncthreads(); + + tile_idx = temp_storage.tile_idx; +#endif + block_offset = SizeT(TILE_ITEMS) * tile_idx; + } + + // Consume a partially-full tile + if (block_offset < num_items) + { + // Consume a partially-full tile + int valid_items = num_items - block_offset; + ConsumeTile(tile_idx, block_offset, valid_items); + } + } + +}; + + +} // CUB namespace +CUB_NS_POSTFIX // Optional outer namespace(s) + diff --git a/lib/kokkos/TPL/cub/device/block/block_reduce_tiles.cuh b/lib/kokkos/TPL/cub/device/block/block_reduce_tiles.cuh new file mode 100755 index 0000000000..a83c098aeb --- /dev/null +++ b/lib/kokkos/TPL/cub/device/block/block_reduce_tiles.cuh @@ -0,0 +1,375 @@ +/****************************************************************************** + * Copyright (c) 2011, Duane Merrill. All rights reserved. + * Copyright (c) 2011-2013, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the NVIDIA CORPORATION nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + ******************************************************************************/ + +/** + * \file + * cub::BlockReduceTiles implements a stateful abstraction of CUDA thread blocks for participating in device-wide reduction. + */ + +#pragma once + +#include + +#include "../../block/block_load.cuh" +#include "../../block/block_reduce.cuh" +#include "../../grid/grid_mapping.cuh" +#include "../../grid/grid_queue.cuh" +#include "../../grid/grid_even_share.cuh" +#include "../../util_vector.cuh" +#include "../../util_namespace.cuh" + + +/// Optional outer namespace(s) +CUB_NS_PREFIX + +/// CUB namespace +namespace cub { + + +/****************************************************************************** + * Tuning policy types + ******************************************************************************/ + +/** + * Tuning policy for BlockReduceTiles + */ +template < + int _BLOCK_THREADS, ///< Threads per thread block + int _ITEMS_PER_THREAD, ///< Items per thread per tile of input + int _VECTOR_LOAD_LENGTH, ///< Number of items per vectorized load + BlockReduceAlgorithm _BLOCK_ALGORITHM, ///< Cooperative block-wide reduction algorithm to use + PtxLoadModifier _LOAD_MODIFIER, ///< PTX load modifier + GridMappingStrategy _GRID_MAPPING> ///< How to map tiles of input onto thread blocks +struct BlockReduceTilesPolicy +{ + enum + { + BLOCK_THREADS = _BLOCK_THREADS, + ITEMS_PER_THREAD = _ITEMS_PER_THREAD, + VECTOR_LOAD_LENGTH = _VECTOR_LOAD_LENGTH, + }; + + static const BlockReduceAlgorithm BLOCK_ALGORITHM = _BLOCK_ALGORITHM; + static const GridMappingStrategy GRID_MAPPING = _GRID_MAPPING; + static const PtxLoadModifier LOAD_MODIFIER = _LOAD_MODIFIER; +}; + + + +/****************************************************************************** + * Thread block abstractions + ******************************************************************************/ + +/** + * \brief BlockReduceTiles implements a stateful abstraction of CUDA thread blocks for participating in device-wide reduction. + * + * Each thread reduces only the values it loads. If \p FIRST_TILE, this + * partial reduction is stored into \p thread_aggregate. Otherwise it is + * accumulated into \p thread_aggregate. + */ +template < + typename BlockReduceTilesPolicy, + typename InputIteratorRA, + typename SizeT, + typename ReductionOp> +struct BlockReduceTiles +{ + + //--------------------------------------------------------------------- + // Types and constants + //--------------------------------------------------------------------- + + typedef typename std::iterator_traits::value_type T; // Type of input iterator + typedef VectorHelper VecHelper; // Helper type for vectorizing loads of T + typedef typename VecHelper::Type VectorT; // Vector of T + + // Constants + enum + { + BLOCK_THREADS = BlockReduceTilesPolicy::BLOCK_THREADS, + ITEMS_PER_THREAD = BlockReduceTilesPolicy::ITEMS_PER_THREAD, + TILE_ITEMS = BLOCK_THREADS * ITEMS_PER_THREAD, + VECTOR_LOAD_LENGTH = BlockReduceTilesPolicy::VECTOR_LOAD_LENGTH, + + // Can vectorize according to the policy if the input iterator is a native pointer to a built-in primitive + CAN_VECTORIZE = (BlockReduceTilesPolicy::VECTOR_LOAD_LENGTH > 1) && + (IsPointer::VALUE) && + (VecHelper::BUILT_IN), + + }; + + static const PtxLoadModifier LOAD_MODIFIER = BlockReduceTilesPolicy::LOAD_MODIFIER; + static const BlockReduceAlgorithm BLOCK_ALGORITHM = BlockReduceTilesPolicy::BLOCK_ALGORITHM; + + // Parameterized BlockReduce primitive + typedef BlockReduce BlockReduceT; + + /// Shared memory type required by this thread block + typedef typename BlockReduceT::TempStorage _TempStorage; + + /// Alias wrapper allowing storage to be unioned + struct TempStorage : Uninitialized<_TempStorage> {}; + + + //--------------------------------------------------------------------- + // Per-thread fields + //--------------------------------------------------------------------- + + T thread_aggregate; ///< Each thread's partial reduction + _TempStorage& temp_storage; ///< Reference to temp_storage + InputIteratorRA d_in; ///< Input data to reduce + ReductionOp reduction_op; ///< Binary reduction operator + int first_tile_size; ///< Size of first tile consumed + bool input_aligned; ///< Whether or not input is vector-aligned + + + //--------------------------------------------------------------------- + // Interface + //--------------------------------------------------------------------- + + /** + * Constructor + */ + __device__ __forceinline__ BlockReduceTiles( + TempStorage& temp_storage, ///< Reference to temp_storage + InputIteratorRA d_in, ///< Input data to reduce + ReductionOp reduction_op) ///< Binary reduction operator + : + temp_storage(temp_storage.Alias()), + d_in(d_in), + reduction_op(reduction_op), + first_tile_size(0), + input_aligned(CAN_VECTORIZE && ((size_t(d_in) & (sizeof(VectorT) - 1)) == 0)) + {} + + + /** + * Process a single tile of input + */ + template + __device__ __forceinline__ void ConsumeTile( + SizeT block_offset, ///< The offset the tile to consume + int valid_items = TILE_ITEMS) ///< The number of valid items in the tile + { + if (FULL_TILE) + { + T stripe_partial; + + // Load full tile + if (input_aligned) + { + // Alias items as an array of VectorT and load it in striped fashion + enum { WORDS = ITEMS_PER_THREAD / VECTOR_LOAD_LENGTH }; + + VectorT vec_items[WORDS]; + + // Load striped into vec items + VectorT* alias_ptr = reinterpret_cast(d_in + block_offset + (threadIdx.x * VECTOR_LOAD_LENGTH)); + + #pragma unroll + for (int i = 0; i < WORDS; ++i) + vec_items[i] = alias_ptr[BLOCK_THREADS * i]; + + // Reduce items within each thread stripe + stripe_partial = ThreadReduce( + reinterpret_cast(vec_items), + reduction_op); + } + else + { + T items[ITEMS_PER_THREAD]; + + // Load items in striped fashion + LoadStriped(threadIdx.x, d_in + block_offset, items); + + // Reduce items within each thread stripe + stripe_partial = ThreadReduce(items, reduction_op); + } + + // Update running thread aggregate + thread_aggregate = (first_tile_size) ? + reduction_op(thread_aggregate, stripe_partial) : // Update + stripe_partial; // Assign + } + else + { + + // Partial tile + int thread_offset = threadIdx.x; + + if (!first_tile_size && (thread_offset < valid_items)) + { + // Assign thread_aggregate + thread_aggregate = ThreadLoad(d_in + block_offset + thread_offset); + thread_offset += BLOCK_THREADS; + } + + while (thread_offset < valid_items) + { + // Update thread aggregate + T item = ThreadLoad(d_in + block_offset + thread_offset); + thread_aggregate = reduction_op(thread_aggregate, item); + thread_offset += BLOCK_THREADS; + } + } + + // Set first tile size if necessary + if (!first_tile_size) + first_tile_size = valid_items; + } + + + //--------------------------------------------------------------------- + // Consume a contiguous segment of tiles + //--------------------------------------------------------------------- + + /** + * \brief Reduce a contiguous segment of input tiles + */ + __device__ __forceinline__ void ConsumeTiles( + SizeT block_offset, ///< [in] Threadblock begin offset (inclusive) + SizeT block_oob, ///< [in] Threadblock end offset (exclusive) + T &block_aggregate) ///< [out] Running total + { + // Consume subsequent full tiles of input + while (block_offset + TILE_ITEMS <= block_oob) + { + ConsumeTile(block_offset); + block_offset += TILE_ITEMS; + } + + // Consume a partially-full tile + if (block_offset < block_oob) + { + int valid_items = block_oob - block_offset; + ConsumeTile(block_offset, valid_items); + } + + // Compute block-wide reduction + block_aggregate = (first_tile_size < TILE_ITEMS) ? + BlockReduceT(temp_storage).Reduce(thread_aggregate, reduction_op, first_tile_size) : + BlockReduceT(temp_storage).Reduce(thread_aggregate, reduction_op); + } + + + /** + * Reduce a contiguous segment of input tiles + */ + __device__ __forceinline__ void ConsumeTiles( + SizeT num_items, ///< [in] Total number of global input items + GridEvenShare &even_share, ///< [in] GridEvenShare descriptor + GridQueue &queue, ///< [in,out] GridQueue descriptor + T &block_aggregate, ///< [out] Running total + Int2Type is_even_share) ///< [in] Marker type indicating this is an even-share mapping + { + // Initialize even-share descriptor for this thread block + even_share.BlockInit(); + + // Consume input tiles + ConsumeTiles(even_share.block_offset, even_share.block_oob, block_aggregate); + } + + + //--------------------------------------------------------------------- + // Dynamically consume tiles + //--------------------------------------------------------------------- + + /** + * Dequeue and reduce tiles of items as part of a inter-block scan + */ + __device__ __forceinline__ void ConsumeTiles( + int num_items, ///< Total number of input items + GridQueue queue, ///< Queue descriptor for assigning tiles of work to thread blocks + T &block_aggregate) ///< [out] Running total + { + // Shared dequeue offset + __shared__ SizeT dequeue_offset; + + // We give each thread block at least one tile of input. + SizeT block_offset = blockIdx.x * TILE_ITEMS; + SizeT even_share_base = gridDim.x * TILE_ITEMS; + + if (block_offset + TILE_ITEMS <= num_items) + { + // Consume full tile of input + ConsumeTile(block_offset); + + // Dequeue more tiles + while (true) + { + // Dequeue a tile of items + if (threadIdx.x == 0) + dequeue_offset = queue.Drain(TILE_ITEMS) + even_share_base; + + __syncthreads(); + + // Grab tile offset and check if we're done with full tiles + block_offset = dequeue_offset; + + __syncthreads(); + + if (block_offset + TILE_ITEMS > num_items) + break; + + // Consume a full tile + ConsumeTile(block_offset); + } + } + + if (block_offset < num_items) + { + int valid_items = num_items - block_offset; + ConsumeTile(block_offset, valid_items); + } + + // Compute block-wide reduction + block_aggregate = (first_tile_size < TILE_ITEMS) ? + BlockReduceT(temp_storage).Reduce(thread_aggregate, reduction_op, first_tile_size) : + BlockReduceT(temp_storage).Reduce(thread_aggregate, reduction_op); + } + + + /** + * Dequeue and reduce tiles of items as part of a inter-block scan + */ + __device__ __forceinline__ void ConsumeTiles( + SizeT num_items, ///< [in] Total number of global input items + GridEvenShare &even_share, ///< [in] GridEvenShare descriptor + GridQueue &queue, ///< [in,out] GridQueue descriptor + T &block_aggregate, ///< [out] Running total + Int2Type is_dynamic) ///< [in] Marker type indicating this is a dynamic mapping + { + ConsumeTiles(num_items, queue, block_aggregate); + } + +}; + + +} // CUB namespace +CUB_NS_POSTFIX // Optional outer namespace(s) + diff --git a/lib/kokkos/TPL/cub/device/block/block_scan_tiles.cuh b/lib/kokkos/TPL/cub/device/block/block_scan_tiles.cuh new file mode 100755 index 0000000000..980220480e --- /dev/null +++ b/lib/kokkos/TPL/cub/device/block/block_scan_tiles.cuh @@ -0,0 +1,509 @@ +/****************************************************************************** + * Copyright (c) 2011, Duane Merrill. All rights reserved. + * Copyright (c) 2011-2013, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the NVIDIA CORPORATION nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + ******************************************************************************/ + +/** + * \file + * cub::BlockScanTiles implements a stateful abstraction of CUDA thread blocks for participating in device-wide prefix scan. + */ + +#pragma once + +#include + +#include "scan_tiles_types.cuh" +#include "../../block/block_load.cuh" +#include "../../block/block_store.cuh" +#include "../../block/block_scan.cuh" +#include "../../grid/grid_queue.cuh" +#include "../../util_namespace.cuh" + +/// Optional outer namespace(s) +CUB_NS_PREFIX + +/// CUB namespace +namespace cub { + + +/****************************************************************************** + * Tuning policy types + ******************************************************************************/ + +/** + * Tuning policy for BlockScanTiles + */ +template < + int _BLOCK_THREADS, + int _ITEMS_PER_THREAD, + BlockLoadAlgorithm _LOAD_ALGORITHM, + bool _LOAD_WARP_TIME_SLICING, + PtxLoadModifier _LOAD_MODIFIER, + BlockStoreAlgorithm _STORE_ALGORITHM, + bool _STORE_WARP_TIME_SLICING, + BlockScanAlgorithm _SCAN_ALGORITHM> +struct BlockScanTilesPolicy +{ + enum + { + BLOCK_THREADS = _BLOCK_THREADS, + ITEMS_PER_THREAD = _ITEMS_PER_THREAD, + LOAD_WARP_TIME_SLICING = _LOAD_WARP_TIME_SLICING, + STORE_WARP_TIME_SLICING = _STORE_WARP_TIME_SLICING, + }; + + static const BlockLoadAlgorithm LOAD_ALGORITHM = _LOAD_ALGORITHM; + static const PtxLoadModifier LOAD_MODIFIER = _LOAD_MODIFIER; + static const BlockStoreAlgorithm STORE_ALGORITHM = _STORE_ALGORITHM; + static const BlockScanAlgorithm SCAN_ALGORITHM = _SCAN_ALGORITHM; +}; + + +/****************************************************************************** + * Thread block abstractions + ******************************************************************************/ + +/** + * \brief BlockScanTiles implements a stateful abstraction of CUDA thread blocks for participating in device-wide prefix scan. + * + * Implements a single-pass "domino" strategy with adaptive prefix lookback. + */ +template < + typename BlockScanTilesPolicy, ///< Tuning policy + typename InputIteratorRA, ///< Input iterator type + typename OutputIteratorRA, ///< Output iterator type + typename ScanOp, ///< Scan functor type + typename Identity, ///< Identity element type (cub::NullType for inclusive scan) + typename SizeT> ///< Offset integer type +struct BlockScanTiles +{ + //--------------------------------------------------------------------- + // Types and constants + //--------------------------------------------------------------------- + + // Data type of input iterator + typedef typename std::iterator_traits::value_type T; + + // Constants + enum + { + INCLUSIVE = Equals::VALUE, // Inclusive scan if no identity type is provided + BLOCK_THREADS = BlockScanTilesPolicy::BLOCK_THREADS, + ITEMS_PER_THREAD = BlockScanTilesPolicy::ITEMS_PER_THREAD, + TILE_ITEMS = BLOCK_THREADS * ITEMS_PER_THREAD, + }; + + // Block load type + typedef BlockLoad< + InputIteratorRA, + BlockScanTilesPolicy::BLOCK_THREADS, + BlockScanTilesPolicy::ITEMS_PER_THREAD, + BlockScanTilesPolicy::LOAD_ALGORITHM, + BlockScanTilesPolicy::LOAD_MODIFIER, + BlockScanTilesPolicy::LOAD_WARP_TIME_SLICING> BlockLoadT; + + // Block store type + typedef BlockStore< + OutputIteratorRA, + BlockScanTilesPolicy::BLOCK_THREADS, + BlockScanTilesPolicy::ITEMS_PER_THREAD, + BlockScanTilesPolicy::STORE_ALGORITHM, + STORE_DEFAULT, + BlockScanTilesPolicy::STORE_WARP_TIME_SLICING> BlockStoreT; + + // Tile status descriptor type + typedef ScanTileDescriptor ScanTileDescriptorT; + + // Block scan type + typedef BlockScan< + T, + BlockScanTilesPolicy::BLOCK_THREADS, + BlockScanTilesPolicy::SCAN_ALGORITHM> BlockScanT; + + // Callback type for obtaining inter-tile prefix during block scan + typedef DeviceScanBlockPrefixOp InterblockPrefixOp; + + // Shared memory type for this threadblock + struct _TempStorage + { + union + { + typename BlockLoadT::TempStorage load; // Smem needed for tile loading + typename BlockStoreT::TempStorage store; // Smem needed for tile storing + struct + { + typename InterblockPrefixOp::TempStorage prefix; // Smem needed for cooperative prefix callback + typename BlockScanT::TempStorage scan; // Smem needed for tile scanning + }; + }; + + SizeT tile_idx; // Shared tile index + }; + + // Alias wrapper allowing storage to be unioned + struct TempStorage : Uninitialized<_TempStorage> {}; + + + //--------------------------------------------------------------------- + // Per-thread fields + //--------------------------------------------------------------------- + + _TempStorage &temp_storage; ///< Reference to temp_storage + InputIteratorRA d_in; ///< Input data + OutputIteratorRA d_out; ///< Output data + ScanOp scan_op; ///< Binary scan operator + Identity identity; ///< Identity element + + + + //--------------------------------------------------------------------- + // Block scan utility methods (first tile) + //--------------------------------------------------------------------- + + /** + * Exclusive scan specialization + */ + template + __device__ __forceinline__ + void ScanBlock(T (&items)[ITEMS_PER_THREAD], _ScanOp scan_op, _Identity identity, T& block_aggregate) + { + BlockScanT(temp_storage.scan).ExclusiveScan(items, items, identity, scan_op, block_aggregate); + } + + /** + * Exclusive sum specialization + */ + template + __device__ __forceinline__ + void ScanBlock(T (&items)[ITEMS_PER_THREAD], Sum scan_op, _Identity identity, T& block_aggregate) + { + BlockScanT(temp_storage.scan).ExclusiveSum(items, items, block_aggregate); + } + + /** + * Inclusive scan specialization + */ + template + __device__ __forceinline__ + void ScanBlock(T (&items)[ITEMS_PER_THREAD], _ScanOp scan_op, NullType identity, T& block_aggregate) + { + BlockScanT(temp_storage.scan).InclusiveScan(items, items, scan_op, block_aggregate); + } + + /** + * Inclusive sum specialization + */ + __device__ __forceinline__ + void ScanBlock(T (&items)[ITEMS_PER_THREAD], Sum scan_op, NullType identity, T& block_aggregate) + { + BlockScanT(temp_storage.scan).InclusiveSum(items, items, block_aggregate); + } + + //--------------------------------------------------------------------- + // Block scan utility methods (subsequent tiles) + //--------------------------------------------------------------------- + + /** + * Exclusive scan specialization (with prefix from predecessors) + */ + template + __device__ __forceinline__ + void ScanBlock(T (&items)[ITEMS_PER_THREAD], _ScanOp scan_op, _Identity identity, T& block_aggregate, PrefixCallback &prefix_op) + { + BlockScanT(temp_storage.scan).ExclusiveScan(items, items, identity, scan_op, block_aggregate, prefix_op); + } + + /** + * Exclusive sum specialization (with prefix from predecessors) + */ + template + __device__ __forceinline__ + void ScanBlock(T (&items)[ITEMS_PER_THREAD], Sum scan_op, _Identity identity, T& block_aggregate, PrefixCallback &prefix_op) + { + BlockScanT(temp_storage.scan).ExclusiveSum(items, items, block_aggregate, prefix_op); + } + + /** + * Inclusive scan specialization (with prefix from predecessors) + */ + template + __device__ __forceinline__ + void ScanBlock(T (&items)[ITEMS_PER_THREAD], _ScanOp scan_op, NullType identity, T& block_aggregate, PrefixCallback &prefix_op) + { + BlockScanT(temp_storage.scan).InclusiveScan(items, items, scan_op, block_aggregate, prefix_op); + } + + /** + * Inclusive sum specialization (with prefix from predecessors) + */ + template + __device__ __forceinline__ + void ScanBlock(T (&items)[ITEMS_PER_THREAD], Sum scan_op, NullType identity, T& block_aggregate, PrefixCallback &prefix_op) + { + BlockScanT(temp_storage.scan).InclusiveSum(items, items, block_aggregate, prefix_op); + } + + //--------------------------------------------------------------------- + // Constructor + //--------------------------------------------------------------------- + + // Constructor + __device__ __forceinline__ + BlockScanTiles( + TempStorage &temp_storage, ///< Reference to temp_storage + InputIteratorRA d_in, ///< Input data + OutputIteratorRA d_out, ///< Output data + ScanOp scan_op, ///< Binary scan operator + Identity identity) ///< Identity element + : + temp_storage(temp_storage.Alias()), + d_in(d_in), + d_out(d_out), + scan_op(scan_op), + identity(identity) + {} + + + //--------------------------------------------------------------------- + // Domino scan + //--------------------------------------------------------------------- + + /** + * Process a tile of input (domino scan) + */ + template + __device__ __forceinline__ void ConsumeTile( + SizeT num_items, ///< Total number of input items + int tile_idx, ///< Tile index + SizeT block_offset, ///< Tile offset + ScanTileDescriptorT *d_tile_status) ///< Global list of tile status + { + // Load items + T items[ITEMS_PER_THREAD]; + + if (FULL_TILE) + BlockLoadT(temp_storage.load).Load(d_in + block_offset, items); + else + BlockLoadT(temp_storage.load).Load(d_in + block_offset, items, num_items - block_offset); + + __syncthreads(); + + T block_aggregate; + if (tile_idx == 0) + { + ScanBlock(items, scan_op, identity, block_aggregate); + + // Update tile status if there are successor tiles + if (FULL_TILE && (threadIdx.x == 0)) + ScanTileDescriptorT::SetPrefix(d_tile_status, block_aggregate); + } + else + { + InterblockPrefixOp prefix_op(d_tile_status, temp_storage.prefix, scan_op, tile_idx); + ScanBlock(items, scan_op, identity, block_aggregate, prefix_op); + } + + __syncthreads(); + + // Store items + if (FULL_TILE) + BlockStoreT(temp_storage.store).Store(d_out + block_offset, items); + else + BlockStoreT(temp_storage.store).Store(d_out + block_offset, items, num_items - block_offset); + } + + /** + * Dequeue and scan tiles of items as part of a domino scan + */ + __device__ __forceinline__ void ConsumeTiles( + int num_items, ///< Total number of input items + GridQueue queue, ///< Queue descriptor for assigning tiles of work to thread blocks + ScanTileDescriptorT *d_tile_status) ///< Global list of tile status + { +#if CUB_PTX_ARCH < 200 + + // No concurrent kernels allowed and blocks are launched in increasing order, so just assign one tile per block (up to 65K blocks) + int tile_idx = blockIdx.x; + SizeT block_offset = SizeT(TILE_ITEMS) * tile_idx; + + if (block_offset + TILE_ITEMS <= num_items) + ConsumeTile(num_items, tile_idx, block_offset, d_tile_status); + else if (block_offset < num_items) + ConsumeTile(num_items, tile_idx, block_offset, d_tile_status); + +#else + + // Get first tile + if (threadIdx.x == 0) + temp_storage.tile_idx = queue.Drain(1); + + __syncthreads(); + + int tile_idx = temp_storage.tile_idx; + SizeT block_offset = SizeT(TILE_ITEMS) * tile_idx; + + while (block_offset + TILE_ITEMS <= num_items) + { + // Consume full tile + ConsumeTile(num_items, tile_idx, block_offset, d_tile_status); + + // Get next tile + if (threadIdx.x == 0) + temp_storage.tile_idx = queue.Drain(1); + + __syncthreads(); + + tile_idx = temp_storage.tile_idx; + block_offset = SizeT(TILE_ITEMS) * tile_idx; + } + + // Consume a partially-full tile + if (block_offset < num_items) + { + ConsumeTile(num_items, tile_idx, block_offset, d_tile_status); + } +#endif + + } + + + //--------------------------------------------------------------------- + // Even-share scan + //--------------------------------------------------------------------- + + /** + * Process a tile of input + */ + template < + bool FULL_TILE, + bool FIRST_TILE> + __device__ __forceinline__ void ConsumeTile( + SizeT block_offset, ///< Tile offset + RunningBlockPrefixOp &prefix_op, ///< Running prefix operator + int valid_items = TILE_ITEMS) ///< Number of valid items in the tile + { + // Load items + T items[ITEMS_PER_THREAD]; + + if (FULL_TILE) + BlockLoadT(temp_storage.load).Load(d_in + block_offset, items); + else + BlockLoadT(temp_storage.load).Load(d_in + block_offset, items, valid_items); + + __syncthreads(); + + // Block scan + T block_aggregate; + if (FIRST_TILE) + { + ScanBlock(items, scan_op, identity, block_aggregate); + prefix_op.running_total = block_aggregate; + } + else + { + ScanBlock(items, scan_op, identity, block_aggregate, prefix_op); + } + + __syncthreads(); + + // Store items + if (FULL_TILE) + BlockStoreT(temp_storage.store).Store(d_out + block_offset, items); + else + BlockStoreT(temp_storage.store).Store(d_out + block_offset, items, valid_items); + } + + + /** + * Scan a consecutive share of input tiles + */ + __device__ __forceinline__ void ConsumeTiles( + SizeT block_offset, ///< [in] Threadblock begin offset (inclusive) + SizeT block_oob) ///< [in] Threadblock end offset (exclusive) + { + RunningBlockPrefixOp prefix_op; + + if (block_offset + TILE_ITEMS <= block_oob) + { + // Consume first tile of input (full) + ConsumeTile(block_offset, prefix_op); + block_offset += TILE_ITEMS; + + // Consume subsequent full tiles of input + while (block_offset + TILE_ITEMS <= block_oob) + { + ConsumeTile(block_offset, prefix_op); + block_offset += TILE_ITEMS; + } + + // Consume a partially-full tile + if (block_offset < block_oob) + { + int valid_items = block_oob - block_offset; + ConsumeTile(block_offset, prefix_op, valid_items); + } + } + else + { + // Consume the first tile of input (partially-full) + int valid_items = block_oob - block_offset; + ConsumeTile(block_offset, prefix_op, valid_items); + } + } + + + /** + * Scan a consecutive share of input tiles, seeded with the specified prefix value + */ + __device__ __forceinline__ void ConsumeTiles( + SizeT block_offset, ///< [in] Threadblock begin offset (inclusive) + SizeT block_oob, ///< [in] Threadblock end offset (exclusive) + T prefix) ///< [in] The prefix to apply to the scan segment + { + RunningBlockPrefixOp prefix_op; + prefix_op.running_total = prefix; + + // Consume full tiles of input + while (block_offset + TILE_ITEMS <= block_oob) + { + ConsumeTile(block_offset, prefix_op); + block_offset += TILE_ITEMS; + } + + // Consume a partially-full tile + if (block_offset < block_oob) + { + int valid_items = block_oob - block_offset; + ConsumeTile(block_offset, prefix_op, valid_items); + } + } + +}; + + +} // CUB namespace +CUB_NS_POSTFIX // Optional outer namespace(s) + diff --git a/lib/kokkos/TPL/cub/device/block/scan_tiles_types.cuh b/lib/kokkos/TPL/cub/device/block/scan_tiles_types.cuh new file mode 100755 index 0000000000..2b933d0af2 --- /dev/null +++ b/lib/kokkos/TPL/cub/device/block/scan_tiles_types.cuh @@ -0,0 +1,318 @@ +/****************************************************************************** + * Copyright (c) 2011, Duane Merrill. All rights reserved. + * Copyright (c) 2011-2013, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the NVIDIA CORPORATION nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + ******************************************************************************/ + +/** + * \file + * Utility types for device-wide scan + */ + +#pragma once + +#include + +#include "../../thread/thread_load.cuh" +#include "../../thread/thread_store.cuh" +#include "../../warp/warp_reduce.cuh" +#include "../../util_namespace.cuh" + +/// Optional outer namespace(s) +CUB_NS_PREFIX + +/// CUB namespace +namespace cub { + + +/** + * Enumerations of tile status + */ +enum ScanTileStatus +{ + SCAN_TILE_OOB, // Out-of-bounds (e.g., padding) + SCAN_TILE_INVALID, // Not yet processed + SCAN_TILE_PARTIAL, // Tile aggregate is available + SCAN_TILE_PREFIX, // Inclusive tile prefix is available +}; + + +/** + * Data type of tile status descriptor. + * + * Specialized for scan status and value types that can be combined into the same + * machine word that can be read/written coherently in a single access. + */ +template < + typename T, + bool SINGLE_WORD = (PowerOfTwo::VALUE && (sizeof(T) <= 8))> +struct ScanTileDescriptor +{ + // Status word type + typedef typename If<(sizeof(T) == 8), + long long, + typename If<(sizeof(T) == 4), + int, + typename If<(sizeof(T) == 2), + short, + char>::Type>::Type>::Type StatusWord; + + // Vector word type + typedef typename If<(sizeof(T) == 8), + longlong2, + typename If<(sizeof(T) == 4), + int2, + typename If<(sizeof(T) == 2), + int, + short>::Type>::Type>::Type VectorWord; + + T value; + StatusWord status; + + static __device__ __forceinline__ void SetPrefix(ScanTileDescriptor *ptr, T prefix) + { + ScanTileDescriptor tile_descriptor; + tile_descriptor.status = SCAN_TILE_PREFIX; + tile_descriptor.value = prefix; + + VectorWord alias; + *reinterpret_cast(&alias) = tile_descriptor; + ThreadStore(reinterpret_cast(ptr), alias); + } + + static __device__ __forceinline__ void SetPartial(ScanTileDescriptor *ptr, T partial) + { + ScanTileDescriptor tile_descriptor; + tile_descriptor.status = SCAN_TILE_PARTIAL; + tile_descriptor.value = partial; + + VectorWord alias; + *reinterpret_cast(&alias) = tile_descriptor; + ThreadStore(reinterpret_cast(ptr), alias); + } + + static __device__ __forceinline__ void WaitForValid( + ScanTileDescriptor *ptr, + int &status, + T &value) + { + ScanTileDescriptor tile_descriptor; + while (true) + { + VectorWord alias = ThreadLoad(reinterpret_cast(ptr)); + + tile_descriptor = *reinterpret_cast(&alias); + if (tile_descriptor.status != SCAN_TILE_INVALID) break; + + __threadfence_block(); + } + + status = tile_descriptor.status; + value = tile_descriptor.value; + } + +}; + + +/** + * Data type of tile status descriptor. + * + * Specialized for scan status and value types that cannot fused into + * the same machine word. + */ +template +struct ScanTileDescriptor +{ + T prefix_value; + T partial_value; + + /// Workaround for the fact that win32 doesn't guarantee 16B alignment 16B values of T + union + { + int status; + Uninitialized padding; + }; + + static __device__ __forceinline__ void SetPrefix(ScanTileDescriptor *ptr, T prefix) + { + ThreadStore(&ptr->prefix_value, prefix); + __threadfence_block(); +// __threadfence(); // __threadfence_block seems sufficient on current architectures to prevent reordeing + ThreadStore(&ptr->status, (int) SCAN_TILE_PREFIX); + + } + + static __device__ __forceinline__ void SetPartial(ScanTileDescriptor *ptr, T partial) + { + ThreadStore(&ptr->partial_value, partial); + __threadfence_block(); +// __threadfence(); // __threadfence_block seems sufficient on current architectures to prevent reordeing + ThreadStore(&ptr->status, (int) SCAN_TILE_PARTIAL); + } + + static __device__ __forceinline__ void WaitForValid( + ScanTileDescriptor *ptr, + int &status, + T &value) + { + while (true) + { + status = ThreadLoad(&ptr->status); + if (status != SCAN_TILE_INVALID) break; + + __threadfence_block(); + } + + value = (status == SCAN_TILE_PARTIAL) ? + ThreadLoad(&ptr->partial_value) : + ThreadLoad(&ptr->prefix_value); + } +}; + + +/** + * Stateful prefix functor that provides the the running prefix for + * the current tile by using the callback warp to wait on on + * aggregates/prefixes from predecessor tiles to become available + */ +template < + typename T, + typename ScanOp> +struct DeviceScanBlockPrefixOp +{ + // Parameterized warp reduce + typedef WarpReduce WarpReduceT; + + // Storage type + typedef typename WarpReduceT::TempStorage _TempStorage; + + // Alias wrapper allowing storage to be unioned + typedef Uninitialized<_TempStorage> TempStorage; + + // Tile status descriptor type + typedef ScanTileDescriptor ScanTileDescriptorT; + + // Fields + ScanTileDescriptorT *d_tile_status; ///< Pointer to array of tile status + _TempStorage &temp_storage; ///< Reference to a warp-reduction instance + ScanOp scan_op; ///< Binary scan operator + int tile_idx; ///< The current tile index + T inclusive_prefix; ///< Inclusive prefix for the tile + + // Constructor + __device__ __forceinline__ + DeviceScanBlockPrefixOp( + ScanTileDescriptorT *d_tile_status, + TempStorage &temp_storage, + ScanOp scan_op, + int tile_idx) : + d_tile_status(d_tile_status), + temp_storage(temp_storage.Alias()), + scan_op(scan_op), + tile_idx(tile_idx) {} + + + // Block until all predecessors within the specified window have non-invalid status + __device__ __forceinline__ + void ProcessWindow( + int predecessor_idx, + int &predecessor_status, + T &window_aggregate) + { + T value; + ScanTileDescriptorT::WaitForValid(d_tile_status + predecessor_idx, predecessor_status, value); + + // Perform a segmented reduction to get the prefix for the current window + int flag = (predecessor_status != SCAN_TILE_PARTIAL); + window_aggregate = WarpReduceT(temp_storage).TailSegmentedReduce(value, flag, scan_op); + } + + + // Prefix functor (called by the first warp) + __device__ __forceinline__ + T operator()(T block_aggregate) + { + // Update our status with our tile-aggregate + if (threadIdx.x == 0) + { + ScanTileDescriptorT::SetPartial(d_tile_status + tile_idx, block_aggregate); + } + + // Wait for the window of predecessor tiles to become valid + int predecessor_idx = tile_idx - threadIdx.x - 1; + int predecessor_status; + T window_aggregate; + ProcessWindow(predecessor_idx, predecessor_status, window_aggregate); + + // The exclusive tile prefix starts out as the current window aggregate + T exclusive_prefix = window_aggregate; + + // Keep sliding the window back until we come across a tile whose inclusive prefix is known + while (WarpAll(predecessor_status != SCAN_TILE_PREFIX)) + { + predecessor_idx -= PtxArchProps::WARP_THREADS; + + // Update exclusive tile prefix with the window prefix + ProcessWindow(predecessor_idx, predecessor_status, window_aggregate); + exclusive_prefix = scan_op(window_aggregate, exclusive_prefix); + } + + // Compute the inclusive tile prefix and update the status for this tile + if (threadIdx.x == 0) + { + inclusive_prefix = scan_op(exclusive_prefix, block_aggregate); + ScanTileDescriptorT::SetPrefix( + d_tile_status + tile_idx, + inclusive_prefix); + } + + // Return exclusive_prefix + return exclusive_prefix; + } +}; + + +// Running scan prefix callback type for single-block scans. +// Maintains a running prefix that can be applied to consecutive +// scan operations. +template +struct RunningBlockPrefixOp +{ + // Running prefix + T running_total; + + // Callback operator. + __device__ T operator()(T block_aggregate) + { + T old_prefix = running_total; + running_total += block_aggregate; + return old_prefix; + } +}; + + +} // CUB namespace +CUB_NS_POSTFIX // Optional outer namespace(s) + diff --git a/lib/kokkos/TPL/cub/device/block/specializations/block_histo_tiles_gatomic.cuh b/lib/kokkos/TPL/cub/device/block/specializations/block_histo_tiles_gatomic.cuh new file mode 100755 index 0000000000..5896dbcf63 --- /dev/null +++ b/lib/kokkos/TPL/cub/device/block/specializations/block_histo_tiles_gatomic.cuh @@ -0,0 +1,184 @@ +/****************************************************************************** + * Copyright (c) 2011, Duane Merrill. All rights reserved. + * Copyright (c) 2011-2013, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the NVIDIA CORPORATION nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + ******************************************************************************/ + +/** + * \file + * cub::BlockHistogramTilesGlobalAtomic implements a stateful abstraction of CUDA thread blocks for histogramming multiple tiles as part of device-wide histogram. + */ + +#pragma once + +#include + +#include "../../../util_type.cuh" +#include "../../../util_namespace.cuh" + +/// Optional outer namespace(s) +CUB_NS_PREFIX + +/// CUB namespace +namespace cub { + + + +/** + * BlockHistogramTilesGlobalAtomic implements a stateful abstraction of CUDA thread blocks for histogramming multiple tiles as part of device-wide histogram using global atomics + */ +template < + typename BlockHistogramTilesPolicy, ///< Tuning policy + int BINS, ///< Number of histogram bins per channel + int CHANNELS, ///< Number of channels interleaved in the input data (may be greater than the number of active channels being histogrammed) + int ACTIVE_CHANNELS, ///< Number of channels actively being histogrammed + typename InputIteratorRA, ///< The input iterator type (may be a simple pointer type). Must have a value type that can be cast as an integer in the range [0..BINS-1] + typename HistoCounter, ///< Integral type for counting sample occurrences per histogram bin + typename SizeT> ///< Integer type for offsets +struct BlockHistogramTilesGlobalAtomic +{ + //--------------------------------------------------------------------- + // Types and constants + //--------------------------------------------------------------------- + + // Sample type + typedef typename std::iterator_traits::value_type SampleT; + + // Constants + enum + { + BLOCK_THREADS = BlockHistogramTilesPolicy::BLOCK_THREADS, + ITEMS_PER_THREAD = BlockHistogramTilesPolicy::ITEMS_PER_THREAD, + TILE_CHANNEL_ITEMS = BLOCK_THREADS * ITEMS_PER_THREAD, + TILE_ITEMS = TILE_CHANNEL_ITEMS * CHANNELS, + }; + + // Shared memory type required by this thread block + typedef NullType TempStorage; + + + //--------------------------------------------------------------------- + // Per-thread fields + //--------------------------------------------------------------------- + + /// Reference to output histograms + HistoCounter* (&d_out_histograms)[ACTIVE_CHANNELS]; + + /// Input data to reduce + InputIteratorRA d_in; + + + //--------------------------------------------------------------------- + // Interface + //--------------------------------------------------------------------- + + /** + * Constructor + */ + __device__ __forceinline__ BlockHistogramTilesGlobalAtomic( + TempStorage &temp_storage, ///< Reference to temp_storage + InputIteratorRA d_in, ///< Input data to reduce + HistoCounter* (&d_out_histograms)[ACTIVE_CHANNELS]) ///< Reference to output histograms + : + d_in(d_in), + d_out_histograms(d_out_histograms) + {} + + + /** + * Process a single tile of input + */ + template + __device__ __forceinline__ void ConsumeTile( + SizeT block_offset, ///< The offset the tile to consume + int valid_items = TILE_ITEMS) ///< The number of valid items in the tile + { + if (FULL_TILE) + { + // Full tile of samples to read and composite + SampleT items[ITEMS_PER_THREAD][CHANNELS]; + + #pragma unroll + for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ITEM++) + { + #pragma unroll + for (int CHANNEL = 0; CHANNEL < CHANNELS; ++CHANNEL) + { + if (CHANNEL < ACTIVE_CHANNELS) + { + items[ITEM][CHANNEL] = d_in[block_offset + (ITEM * BLOCK_THREADS * CHANNELS) + (threadIdx.x * CHANNELS) + CHANNEL]; + } + } + } + + __threadfence_block(); + + #pragma unroll + for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ITEM++) + { + #pragma unroll + for (int CHANNEL = 0; CHANNEL < CHANNELS; ++CHANNEL) + { + if (CHANNEL < ACTIVE_CHANNELS) + { + atomicAdd(d_out_histograms[CHANNEL] + items[ITEM][CHANNEL], 1); + } + } + } + } + else + { + // Only a partially-full tile of samples to read and composite + int bounds = valid_items - (threadIdx.x * CHANNELS); + + #pragma unroll + for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ++ITEM) + { + #pragma unroll + for (int CHANNEL = 0; CHANNEL < CHANNELS; ++CHANNEL) + { + if (((ACTIVE_CHANNELS == CHANNELS) || (CHANNEL < ACTIVE_CHANNELS)) && ((ITEM * BLOCK_THREADS * CHANNELS) + CHANNEL < bounds)) + { + SampleT item = d_in[block_offset + (ITEM * BLOCK_THREADS * CHANNELS) + (threadIdx.x * CHANNELS) + CHANNEL]; + atomicAdd(d_out_histograms[CHANNEL] + item, 1); + } + } + } + + } + } + + + /** + * Aggregate results into output + */ + __device__ __forceinline__ void AggregateOutput() + {} +}; + + +} // CUB namespace +CUB_NS_POSTFIX // Optional outer namespace(s) + diff --git a/lib/kokkos/TPL/cub/device/block/specializations/block_histo_tiles_satomic.cuh b/lib/kokkos/TPL/cub/device/block/specializations/block_histo_tiles_satomic.cuh new file mode 100755 index 0000000000..c55d78953c --- /dev/null +++ b/lib/kokkos/TPL/cub/device/block/specializations/block_histo_tiles_satomic.cuh @@ -0,0 +1,237 @@ +/****************************************************************************** + * Copyright (c) 2011, Duane Merrill. All rights reserved. + * Copyright (c) 2011-2013, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the NVIDIA CORPORATION nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + ******************************************************************************/ + +/** + * \file + * cub::BlockHistogramTilesSharedAtomic implements a stateful abstraction of CUDA thread blocks for histogramming multiple tiles as part of device-wide histogram using shared atomics + */ + +#pragma once + +#include + +#include "../../../util_type.cuh" +#include "../../../util_namespace.cuh" + +/// Optional outer namespace(s) +CUB_NS_PREFIX + +/// CUB namespace +namespace cub { + + +/** + * BlockHistogramTilesSharedAtomic implements a stateful abstraction of CUDA thread blocks for histogramming multiple tiles as part of device-wide histogram using shared atomics + */ +template < + typename BlockHistogramTilesPolicy, ///< Tuning policy + int BINS, ///< Number of histogram bins + int CHANNELS, ///< Number of channels interleaved in the input data (may be greater than the number of active channels being histogrammed) + int ACTIVE_CHANNELS, ///< Number of channels actively being histogrammed + typename InputIteratorRA, ///< The input iterator type (may be a simple pointer type). Must have a value type that can be cast as an integer in the range [0..BINS-1] + typename HistoCounter, ///< Integral type for counting sample occurrences per histogram bin + typename SizeT> ///< Integer type for offsets +struct BlockHistogramTilesSharedAtomic +{ + //--------------------------------------------------------------------- + // Types and constants + //--------------------------------------------------------------------- + + // Sample type + typedef typename std::iterator_traits::value_type SampleT; + + // Constants + enum + { + BLOCK_THREADS = BlockHistogramTilesPolicy::BLOCK_THREADS, + ITEMS_PER_THREAD = BlockHistogramTilesPolicy::ITEMS_PER_THREAD, + TILE_CHANNEL_ITEMS = BLOCK_THREADS * ITEMS_PER_THREAD, + TILE_ITEMS = TILE_CHANNEL_ITEMS * CHANNELS, + }; + + /// Shared memory type required by this thread block + struct _TempStorage + { + HistoCounter histograms[ACTIVE_CHANNELS][BINS + 1]; // One word of padding between channel histograms to prevent warps working on different histograms from hammering on the same bank + }; + + + /// Alias wrapper allowing storage to be unioned + struct TempStorage : Uninitialized<_TempStorage> {}; + + + //--------------------------------------------------------------------- + // Per-thread fields + //--------------------------------------------------------------------- + + /// Reference to temp_storage + _TempStorage &temp_storage; + + /// Reference to output histograms + HistoCounter* (&d_out_histograms)[ACTIVE_CHANNELS]; + + /// Input data to reduce + InputIteratorRA d_in; + + + //--------------------------------------------------------------------- + // Interface + //--------------------------------------------------------------------- + + /** + * Constructor + */ + __device__ __forceinline__ BlockHistogramTilesSharedAtomic( + TempStorage &temp_storage, ///< Reference to temp_storage + InputIteratorRA d_in, ///< Input data to reduce + HistoCounter* (&d_out_histograms)[ACTIVE_CHANNELS]) ///< Reference to output histograms + : + temp_storage(temp_storage.Alias()), + d_in(d_in), + d_out_histograms(d_out_histograms) + { + // Initialize histogram bin counts to zeros + #pragma unroll + for (int CHANNEL = 0; CHANNEL < ACTIVE_CHANNELS; ++CHANNEL) + { + int histo_offset = 0; + + #pragma unroll + for(; histo_offset + BLOCK_THREADS <= BINS; histo_offset += BLOCK_THREADS) + { + this->temp_storage.histograms[CHANNEL][histo_offset + threadIdx.x] = 0; + } + // Finish up with guarded initialization if necessary + if ((BINS % BLOCK_THREADS != 0) && (histo_offset + threadIdx.x < BINS)) + { + this->temp_storage.histograms[CHANNEL][histo_offset + threadIdx.x] = 0; + } + } + } + + + /** + * Process a single tile of input + */ + template + __device__ __forceinline__ void ConsumeTile( + SizeT block_offset, ///< The offset the tile to consume + int valid_items = TILE_ITEMS) ///< The number of valid items in the tile + { + if (FULL_TILE) + { + // Full tile of samples to read and composite + SampleT items[ITEMS_PER_THREAD][CHANNELS]; + + #pragma unroll + for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ITEM++) + { + #pragma unroll + for (int CHANNEL = 0; CHANNEL < CHANNELS; ++CHANNEL) + { + if (CHANNEL < ACTIVE_CHANNELS) + { + items[ITEM][CHANNEL] = d_in[block_offset + (ITEM * BLOCK_THREADS * CHANNELS) + (threadIdx.x * CHANNELS) + CHANNEL]; + } + } + } + + __threadfence_block(); + + #pragma unroll + for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ITEM++) + { + #pragma unroll + for (int CHANNEL = 0; CHANNEL < CHANNELS; ++CHANNEL) + { + if (CHANNEL < ACTIVE_CHANNELS) + { + atomicAdd(temp_storage.histograms[CHANNEL] + items[ITEM][CHANNEL], 1); + } + } + } + + __threadfence_block(); + } + else + { + // Only a partially-full tile of samples to read and composite + int bounds = valid_items - (threadIdx.x * CHANNELS); + + #pragma unroll + for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ++ITEM) + { + #pragma unroll + for (int CHANNEL = 0; CHANNEL < CHANNELS; ++CHANNEL) + { + if (((ACTIVE_CHANNELS == CHANNELS) || (CHANNEL < ACTIVE_CHANNELS)) && ((ITEM * BLOCK_THREADS * CHANNELS) + CHANNEL < bounds)) + { + SampleT item = d_in[block_offset + (ITEM * BLOCK_THREADS * CHANNELS) + (threadIdx.x * CHANNELS) + CHANNEL]; + atomicAdd(temp_storage.histograms[CHANNEL] + item, 1); + } + } + } + + } + } + + + /** + * Aggregate results into output + */ + __device__ __forceinline__ void AggregateOutput() + { + // Barrier to ensure shared memory histograms are coherent + __syncthreads(); + + // Copy shared memory histograms to output + #pragma unroll + for (int CHANNEL = 0; CHANNEL < ACTIVE_CHANNELS; ++CHANNEL) + { + int channel_offset = (blockIdx.x * BINS); + int histo_offset = 0; + + #pragma unroll + for(; histo_offset + BLOCK_THREADS <= BINS; histo_offset += BLOCK_THREADS) + { + d_out_histograms[CHANNEL][channel_offset + histo_offset + threadIdx.x] = temp_storage.histograms[CHANNEL][histo_offset + threadIdx.x]; + } + // Finish up with guarded initialization if necessary + if ((BINS % BLOCK_THREADS != 0) && (histo_offset + threadIdx.x < BINS)) + { + d_out_histograms[CHANNEL][channel_offset + histo_offset + threadIdx.x] = temp_storage.histograms[CHANNEL][histo_offset + threadIdx.x]; + } + } + } +}; + + + +} // CUB namespace +CUB_NS_POSTFIX // Optional outer namespace(s) + diff --git a/lib/kokkos/TPL/cub/device/block/specializations/block_histo_tiles_sort.cuh b/lib/kokkos/TPL/cub/device/block/specializations/block_histo_tiles_sort.cuh new file mode 100755 index 0000000000..0f821309cb --- /dev/null +++ b/lib/kokkos/TPL/cub/device/block/specializations/block_histo_tiles_sort.cuh @@ -0,0 +1,364 @@ +/****************************************************************************** + * Copyright (c) 2011, Duane Merrill. All rights reserved. + * Copyright (c) 2011-2013, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the NVIDIA CORPORATION nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + ******************************************************************************/ + +/** + * \file + * cub::BlockHistogramTilesSort implements a stateful abstraction of CUDA thread blocks for histogramming multiple tiles as part of device-wide histogram using local sorting + */ + +#pragma once + +#include + +#include "../../../block/block_radix_sort.cuh" +#include "../../../block/block_discontinuity.cuh" +#include "../../../util_namespace.cuh" + +/// Optional outer namespace(s) +CUB_NS_PREFIX + +/// CUB namespace +namespace cub { + + +/** + * BlockHistogramTilesSort implements a stateful abstraction of CUDA thread blocks for histogramming multiple tiles as part of device-wide histogram using local sorting + */ +template < + typename BlockHistogramTilesPolicy, ///< Tuning policy + int BINS, ///< Number of histogram bins per channel + int CHANNELS, ///< Number of channels interleaved in the input data (may be greater than the number of active channels being histogrammed) + int ACTIVE_CHANNELS, ///< Number of channels actively being histogrammed + typename InputIteratorRA, ///< The input iterator type (may be a simple pointer type). Must have a value type that can be cast as an integer in the range [0..BINS-1] + typename HistoCounter, ///< Integral type for counting sample occurrences per histogram bin + typename SizeT> ///< Integer type for offsets +struct BlockHistogramTilesSort +{ + //--------------------------------------------------------------------- + // Types and constants + //--------------------------------------------------------------------- + + // Sample type + typedef typename std::iterator_traits::value_type SampleT; + + // Constants + enum + { + BLOCK_THREADS = BlockHistogramTilesPolicy::BLOCK_THREADS, + ITEMS_PER_THREAD = BlockHistogramTilesPolicy::ITEMS_PER_THREAD, + TILE_CHANNEL_ITEMS = BLOCK_THREADS * ITEMS_PER_THREAD, + TILE_ITEMS = TILE_CHANNEL_ITEMS * CHANNELS, + + STRIPED_COUNTERS_PER_THREAD = (BINS + BLOCK_THREADS - 1) / BLOCK_THREADS, + }; + + // Parameterize BlockRadixSort type for our thread block + typedef BlockRadixSort BlockRadixSortT; + + // Parameterize BlockDiscontinuity type for our thread block + typedef BlockDiscontinuity BlockDiscontinuityT; + + /// Shared memory type required by this thread block + union _TempStorage + { + // Storage for sorting bin values + typename BlockRadixSortT::TempStorage sort; + + struct + { + // Storage for detecting discontinuities in the tile of sorted bin values + typename BlockDiscontinuityT::TempStorage flag; + + // Storage for noting begin/end offsets of bin runs in the tile of sorted bin values + int run_begin[BLOCK_THREADS * STRIPED_COUNTERS_PER_THREAD]; + int run_end[BLOCK_THREADS * STRIPED_COUNTERS_PER_THREAD]; + }; + }; + + + /// Alias wrapper allowing storage to be unioned + struct TempStorage : Uninitialized<_TempStorage> {}; + + + // Discontinuity functor + struct DiscontinuityOp + { + // Reference to temp_storage + _TempStorage &temp_storage; + + // Constructor + __device__ __forceinline__ DiscontinuityOp(_TempStorage &temp_storage) : + temp_storage(temp_storage) + {} + + // Discontinuity predicate + __device__ __forceinline__ bool operator()(const SampleT &a, const SampleT &b, int b_index) + { + if (a != b) + { + // Note the begin/end offsets in shared storage + temp_storage.run_begin[b] = b_index; + temp_storage.run_end[a] = b_index; + + return true; + } + else + { + return false; + } + } + }; + + + //--------------------------------------------------------------------- + // Per-thread fields + //--------------------------------------------------------------------- + + /// Reference to temp_storage + _TempStorage &temp_storage; + + /// Histogram counters striped across threads + HistoCounter thread_counters[ACTIVE_CHANNELS][STRIPED_COUNTERS_PER_THREAD]; + + /// Reference to output histograms + HistoCounter* (&d_out_histograms)[ACTIVE_CHANNELS]; + + /// Input data to reduce + InputIteratorRA d_in; + + + //--------------------------------------------------------------------- + // Interface + //--------------------------------------------------------------------- + + /** + * Constructor + */ + __device__ __forceinline__ BlockHistogramTilesSort( + TempStorage &temp_storage, ///< Reference to temp_storage + InputIteratorRA d_in, ///< Input data to reduce + HistoCounter* (&d_out_histograms)[ACTIVE_CHANNELS]) ///< Reference to output histograms + : + temp_storage(temp_storage.Alias()), + d_in(d_in), + d_out_histograms(d_out_histograms) + { + // Initialize histogram counters striped across threads + #pragma unroll + for (int CHANNEL = 0; CHANNEL < ACTIVE_CHANNELS; ++CHANNEL) + { + #pragma unroll + for (int COUNTER = 0; COUNTER < STRIPED_COUNTERS_PER_THREAD; ++COUNTER) + { + thread_counters[CHANNEL][COUNTER] = 0; + } + } + } + + + /** + * Composite a tile of input items + */ + __device__ __forceinline__ void Composite( + SampleT (&items)[ITEMS_PER_THREAD], ///< Tile of samples + HistoCounter thread_counters[STRIPED_COUNTERS_PER_THREAD]) ///< Histogram counters striped across threads + { + // Sort bytes in blocked arrangement + BlockRadixSortT(temp_storage.sort).Sort(items); + + __syncthreads(); + + // Initialize the shared memory's run_begin and run_end for each bin + #pragma unroll + for (int COUNTER = 0; COUNTER < STRIPED_COUNTERS_PER_THREAD; ++COUNTER) + { + temp_storage.run_begin[(COUNTER * BLOCK_THREADS) + threadIdx.x] = TILE_CHANNEL_ITEMS; + temp_storage.run_end[(COUNTER * BLOCK_THREADS) + threadIdx.x] = TILE_CHANNEL_ITEMS; + } + + __syncthreads(); + + // Note the begin/end run offsets of bin runs in the sorted tile + int flags[ITEMS_PER_THREAD]; // unused + DiscontinuityOp flag_op(temp_storage); + BlockDiscontinuityT(temp_storage.flag).FlagHeads(flags, items, flag_op); + + // Update begin for first item + if (threadIdx.x == 0) temp_storage.run_begin[items[0]] = 0; + + __syncthreads(); + + // Composite into histogram + // Initialize the shared memory's run_begin and run_end for each bin + #pragma unroll + for (int COUNTER = 0; COUNTER < STRIPED_COUNTERS_PER_THREAD; ++COUNTER) + { + int bin = (COUNTER * BLOCK_THREADS) + threadIdx.x; + HistoCounter run_length = temp_storage.run_end[bin] - temp_storage.run_begin[bin]; + + thread_counters[COUNTER] += run_length; + } + } + + + /** + * Process one channel within a tile. + */ + template + __device__ __forceinline__ void ConsumeTileChannel( + int channel, + SizeT block_offset, + int valid_items) + { + // Load items in striped fashion + if (FULL_TILE) + { + // Full tile of samples to read and composite + SampleT items[ITEMS_PER_THREAD]; + + // Unguarded loads + #pragma unroll + for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ITEM++) + { + items[ITEM] = d_in[channel + block_offset + (ITEM * BLOCK_THREADS * CHANNELS) + (threadIdx.x * CHANNELS)]; + } + + // Composite our histogram data + Composite(items, thread_counters[channel]); + } + else + { + // Only a partially-full tile of samples to read and composite + SampleT items[ITEMS_PER_THREAD]; + + // Assign our tid as the bin for out-of-bounds items (to give an even distribution), and keep track of how oob items to subtract out later + int bounds = (valid_items - (threadIdx.x * CHANNELS)); + + #pragma unroll + for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ITEM++) + { + items[ITEM] = ((ITEM * BLOCK_THREADS * CHANNELS) < bounds) ? + d_in[channel + block_offset + (ITEM * BLOCK_THREADS * CHANNELS) + (threadIdx.x * CHANNELS)] : + 0; + } + + // Composite our histogram data + Composite(items, thread_counters[channel]); + + __syncthreads(); + + // Correct the overcounting in the zero-bin from invalid (out-of-bounds) items + if (threadIdx.x == 0) + { + int extra = (TILE_ITEMS - valid_items) / CHANNELS; + thread_counters[channel][0] -= extra; + } + } + } + + + /** + * Template iteration over channels (to silence not-unrolled warnings for SM10-13). Inductive step. + */ + template + struct IterateChannels + { + /** + * Process one channel within a tile. + */ + static __device__ __forceinline__ void ConsumeTileChannel( + BlockHistogramTilesSort *cta, + SizeT block_offset, + int valid_items) + { + __syncthreads(); + + cta->ConsumeTileChannel(CHANNEL, block_offset, valid_items); + + IterateChannels::ConsumeTileChannel(cta, block_offset, valid_items); + } + }; + + + /** + * Template iteration over channels (to silence not-unrolled warnings for SM10-13). Base step. + */ + template + struct IterateChannels + { + static __device__ __forceinline__ void ConsumeTileChannel(BlockHistogramTilesSort *cta, SizeT block_offset, int valid_items) {} + }; + + + /** + * Process a single tile of input + */ + template + __device__ __forceinline__ void ConsumeTile( + SizeT block_offset, ///< The offset the tile to consume + int valid_items = TILE_ITEMS) ///< The number of valid items in the tile + { + // First channel + ConsumeTileChannel(0, block_offset, valid_items); + + // Iterate through remaining channels + IterateChannels::ConsumeTileChannel(this, block_offset, valid_items); + } + + + /** + * Aggregate results into output + */ + __device__ __forceinline__ void AggregateOutput() + { + // Copy counters striped across threads into the histogram output + #pragma unroll + for (int CHANNEL = 0; CHANNEL < ACTIVE_CHANNELS; ++CHANNEL) + { + int channel_offset = (blockIdx.x * BINS); + + #pragma unroll + for (int COUNTER = 0; COUNTER < STRIPED_COUNTERS_PER_THREAD; ++COUNTER) + { + int bin = (COUNTER * BLOCK_THREADS) + threadIdx.x; + + if ((STRIPED_COUNTERS_PER_THREAD * BLOCK_THREADS == BINS) || (bin < BINS)) + { + d_out_histograms[CHANNEL][channel_offset + bin] = thread_counters[CHANNEL][COUNTER]; + } + } + } + } +}; + + + + +} // CUB namespace +CUB_NS_POSTFIX // Optional outer namespace(s) + diff --git a/lib/kokkos/TPL/cub/device/device_histogram.cuh b/lib/kokkos/TPL/cub/device/device_histogram.cuh new file mode 100755 index 0000000000..6f5a74d1f8 --- /dev/null +++ b/lib/kokkos/TPL/cub/device/device_histogram.cuh @@ -0,0 +1,1062 @@ + +/****************************************************************************** + * Copyright (c) 2011, Duane Merrill. All rights reserved. + * Copyright (c) 2011-2013, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the NVIDIA CORPORATION nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + ******************************************************************************/ + +/** + * \file + * cub::DeviceHistogram provides device-wide parallel operations for constructing histogram(s) from samples data residing within global memory. + */ + +#pragma once + +#include +#include + +#include "block/block_histo_tiles.cuh" +#include "../grid/grid_even_share.cuh" +#include "../grid/grid_queue.cuh" +#include "../util_debug.cuh" +#include "../util_device.cuh" +#include "../util_namespace.cuh" + +/// Optional outer namespace(s) +CUB_NS_PREFIX + +/// CUB namespace +namespace cub { + + +/****************************************************************************** + * Kernel entry points + *****************************************************************************/ + +#ifndef DOXYGEN_SHOULD_SKIP_THIS // Do not document + + +/** + * Initialization pass kernel entry point (multi-block). Prepares queue descriptors zeroes global counters. + */ +template < + int BINS, ///< Number of histogram bins per channel + int ACTIVE_CHANNELS, ///< Number of channels actively being histogrammed + typename SizeT, ///< Integer type used for global array indexing + typename HistoCounter> ///< Integral type for counting sample occurrences per histogram bin +__launch_bounds__ (BINS, 1) +__global__ void InitHistoKernel( + GridQueue grid_queue, ///< [in] Descriptor for performing dynamic mapping of tile data to thread blocks + ArrayWrapper d_out_histograms, ///< [out] Histogram counter data having logical dimensions HistoCounter[ACTIVE_CHANNELS][BINS] + SizeT num_samples) ///< [in] Total number of samples \p d_samples for all channels +{ + d_out_histograms.array[blockIdx.x][threadIdx.x] = 0; + if (threadIdx.x == 0) grid_queue.ResetDrain(num_samples); +} + + +/** + * Histogram pass kernel entry point (multi-block). Computes privatized histograms, one per thread block. + */ +template < + typename BlockHistogramTilesPolicy, ///< Tuning policy for cub::BlockHistogramTiles abstraction + int BINS, ///< Number of histogram bins per channel + int CHANNELS, ///< Number of channels interleaved in the input data (may be greater than the number of channels being actively histogrammed) + int ACTIVE_CHANNELS, ///< Number of channels actively being histogrammed + typename InputIteratorRA, ///< The input iterator type (may be a simple pointer type). Must have a value type that is assignable to unsigned char + typename HistoCounter, ///< Integral type for counting sample occurrences per histogram bin + typename SizeT> ///< Integer type used for global array indexing +__launch_bounds__ (int(BlockHistogramTilesPolicy::BLOCK_THREADS), BlockHistogramTilesPolicy::SM_OCCUPANCY) +__global__ void MultiBlockHistogramKernel( + InputIteratorRA d_samples, ///< [in] Array of sample data. The samples from different channels are assumed to be interleaved (e.g., an array of 32b pixels where each pixel consists of four RGBA 8b samples). + ArrayWrapper d_out_histograms, ///< [out] Histogram counter data having logical dimensions HistoCounter[ACTIVE_CHANNELS][gridDim.x][BINS] + SizeT num_samples, ///< [in] Total number of samples \p d_samples for all channels + GridEvenShare even_share, ///< [in] Descriptor for how to map an even-share of tiles across thread blocks + GridQueue queue) ///< [in] Descriptor for performing dynamic mapping of tile data to thread blocks +{ + // Constants + enum + { + BLOCK_THREADS = BlockHistogramTilesPolicy::BLOCK_THREADS, + ITEMS_PER_THREAD = BlockHistogramTilesPolicy::ITEMS_PER_THREAD, + TILE_SIZE = BLOCK_THREADS * ITEMS_PER_THREAD, + }; + + // Thread block type for compositing input tiles + typedef BlockHistogramTiles BlockHistogramTilesT; + + // Shared memory for BlockHistogramTiles + __shared__ typename BlockHistogramTilesT::TempStorage temp_storage; + + // Consume input tiles + BlockHistogramTilesT(temp_storage, d_samples, d_out_histograms.array).ConsumeTiles( + num_samples, + even_share, + queue, + Int2Type()); +} + + +/** + * Block-aggregation pass kernel entry point (single-block). Aggregates privatized threadblock histograms from a previous multi-block histogram pass. + */ +template < + int BINS, ///< Number of histogram bins per channel + int ACTIVE_CHANNELS, ///< Number of channels actively being histogrammed + typename HistoCounter> ///< Integral type for counting sample occurrences per histogram bin +__launch_bounds__ (BINS, 1) +__global__ void AggregateHistoKernel( + HistoCounter* d_block_histograms, ///< [in] Histogram counter data having logical dimensions HistoCounter[ACTIVE_CHANNELS][num_threadblocks][BINS] + ArrayWrapper d_out_histograms, ///< [out] Histogram counter data having logical dimensions HistoCounter[ACTIVE_CHANNELS][BINS] + int num_threadblocks) ///< [in] Number of threadblock histograms per channel in \p d_block_histograms +{ + // Accumulate threadblock-histograms from the channel + HistoCounter bin_aggregate = 0; + + int block_offset = blockIdx.x * (num_threadblocks * BINS); + int block_oob = block_offset + (num_threadblocks * BINS); + +#if CUB_PTX_ARCH >= 200 + #pragma unroll 32 +#endif + while (block_offset < block_oob) + { + bin_aggregate += d_block_histograms[block_offset + threadIdx.x]; + block_offset += BINS; + } + + // Output + d_out_histograms.array[blockIdx.x][threadIdx.x] = bin_aggregate; +} + +#endif // DOXYGEN_SHOULD_SKIP_THIS + + + +/****************************************************************************** + * DeviceHistogram + *****************************************************************************/ + +/** + * \brief DeviceHistogram provides device-wide parallel operations for constructing histogram(s) from samples data residing within global memory. ![](histogram_logo.png) + * \ingroup DeviceModule + * + * \par Overview + * A histogram + * counts the number of observations that fall into each of the disjoint categories (known as bins). + * + * \par Usage Considerations + * \cdp_class{DeviceHistogram} + * + * \par Performance + * + * \image html histo_perf.png + * + */ +struct DeviceHistogram +{ +#ifndef DOXYGEN_SHOULD_SKIP_THIS // Do not document + + /****************************************************************************** + * Constants and typedefs + ******************************************************************************/ + + /// Generic structure for encapsulating dispatch properties. Mirrors the constants within BlockHistogramTilesPolicy. + struct KernelDispachParams + { + // Policy fields + int block_threads; + int items_per_thread; + BlockHistogramTilesAlgorithm block_algorithm; + GridMappingStrategy grid_mapping; + int subscription_factor; + + // Derived fields + int channel_tile_size; + + template + __host__ __device__ __forceinline__ + void Init(int subscription_factor = 1) + { + block_threads = BlockHistogramTilesPolicy::BLOCK_THREADS; + items_per_thread = BlockHistogramTilesPolicy::ITEMS_PER_THREAD; + block_algorithm = BlockHistogramTilesPolicy::GRID_ALGORITHM; + grid_mapping = BlockHistogramTilesPolicy::GRID_MAPPING; + this->subscription_factor = subscription_factor; + + channel_tile_size = block_threads * items_per_thread; + } + + __host__ __device__ __forceinline__ + void Print() + { + printf("%d, %d, %d, %d, %d", + block_threads, + items_per_thread, + block_algorithm, + grid_mapping, + subscription_factor); + } + + }; + + + /****************************************************************************** + * Tuning policies + ******************************************************************************/ + + /// Specializations of tuned policy types for different PTX architectures + template < + int CHANNELS, + int ACTIVE_CHANNELS, + BlockHistogramTilesAlgorithm GRID_ALGORITHM, + int ARCH> + struct TunedPolicies; + + /// SM35 tune + template + struct TunedPolicies + { + typedef BlockHistogramTilesPolicy< + (GRID_ALGORITHM == GRID_HISTO_SORT) ? 128 : 256, + (GRID_ALGORITHM == GRID_HISTO_SORT) ? 12 : (30 / ACTIVE_CHANNELS), + GRID_ALGORITHM, + (GRID_ALGORITHM == GRID_HISTO_SORT) ? GRID_MAPPING_DYNAMIC : GRID_MAPPING_EVEN_SHARE, + (GRID_ALGORITHM == GRID_HISTO_SORT) ? 8 : 1> MultiBlockPolicy; + enum { SUBSCRIPTION_FACTOR = 7 }; + }; + + /// SM30 tune + template + struct TunedPolicies + { + typedef BlockHistogramTilesPolicy< + 128, + (GRID_ALGORITHM == GRID_HISTO_SORT) ? 20 : (22 / ACTIVE_CHANNELS), + GRID_ALGORITHM, + (GRID_ALGORITHM == GRID_HISTO_SORT) ? GRID_MAPPING_DYNAMIC : GRID_MAPPING_EVEN_SHARE, + 1> MultiBlockPolicy; + enum { SUBSCRIPTION_FACTOR = 1 }; + }; + + /// SM20 tune + template + struct TunedPolicies + { + typedef BlockHistogramTilesPolicy< + 128, + (GRID_ALGORITHM == GRID_HISTO_SORT) ? 21 : (23 / ACTIVE_CHANNELS), + GRID_ALGORITHM, + GRID_MAPPING_DYNAMIC, + 1> MultiBlockPolicy; + enum { SUBSCRIPTION_FACTOR = 1 }; + }; + + /// SM10 tune + template + struct TunedPolicies + { + typedef BlockHistogramTilesPolicy< + 128, + 7, + GRID_HISTO_SORT, // (use sort regardless because atomics are perf-useless) + GRID_MAPPING_EVEN_SHARE, + 1> MultiBlockPolicy; + enum { SUBSCRIPTION_FACTOR = 1 }; + }; + + + /// Tuning policy for the PTX architecture that DeviceHistogram operations will get dispatched to + template < + int CHANNELS, + int ACTIVE_CHANNELS, + BlockHistogramTilesAlgorithm GRID_ALGORITHM> + struct PtxDefaultPolicies + { + static const int PTX_TUNE_ARCH = (CUB_PTX_ARCH >= 350) ? + 350 : + (CUB_PTX_ARCH >= 300) ? + 300 : + (CUB_PTX_ARCH >= 200) ? + 200 : + 100; + + // Tuned policy set for the current PTX compiler pass + typedef TunedPolicies PtxTunedPolicies; + + // Subscription factor for the current PTX compiler pass + static const int SUBSCRIPTION_FACTOR = PtxTunedPolicies::SUBSCRIPTION_FACTOR; + + // MultiBlockPolicy that opaquely derives from the specialization corresponding to the current PTX compiler pass + struct MultiBlockPolicy : PtxTunedPolicies::MultiBlockPolicy {}; + + /** + * Initialize dispatch params with the policies corresponding to the PTX assembly we will use + */ + static void InitDispatchParams(int ptx_version, KernelDispachParams &multi_block_dispatch_params) + { + if (ptx_version >= 350) + { + typedef TunedPolicies TunedPolicies; + multi_block_dispatch_params.Init(TunedPolicies::SUBSCRIPTION_FACTOR); + } + else if (ptx_version >= 300) + { + typedef TunedPolicies TunedPolicies; + multi_block_dispatch_params.Init(TunedPolicies::SUBSCRIPTION_FACTOR); + } + else if (ptx_version >= 200) + { + typedef TunedPolicies TunedPolicies; + multi_block_dispatch_params.Init(TunedPolicies::SUBSCRIPTION_FACTOR); + } + else + { + typedef TunedPolicies TunedPolicies; + multi_block_dispatch_params.Init(TunedPolicies::SUBSCRIPTION_FACTOR); + } + } + }; + + + /****************************************************************************** + * Utility methods + ******************************************************************************/ + + /** + * Internal dispatch routine for invoking device-wide, multi-channel, histogram + */ + template < + int BINS, ///< Number of histogram bins per channel + int CHANNELS, ///< Number of channels interleaved in the input data (may be greater than the number of channels being actively histogrammed) + int ACTIVE_CHANNELS, ///< Number of channels actively being histogrammed + typename InitHistoKernelPtr, ///< Function type of cub::InitHistoKernel + typename MultiBlockHistogramKernelPtr, ///< Function type of cub::MultiBlockHistogramKernel + typename AggregateHistoKernelPtr, ///< Function type of cub::AggregateHistoKernel + typename InputIteratorRA, ///< The input iterator type (may be a simple pointer type). Must have a value type that is assignable to unsigned char + typename HistoCounter, ///< Integral type for counting sample occurrences per histogram bin + typename SizeT> ///< Integer type used for global array indexing + __host__ __device__ __forceinline__ + static cudaError_t Dispatch( + void *d_temp_storage, ///< [in] %Device allocation of temporary storage. When NULL, the required allocation size is returned in \p temp_storage_bytes and no work is done. + size_t &temp_storage_bytes, ///< [in,out] Size in bytes of \p d_temp_storage allocation. + InitHistoKernelPtr init_kernel, ///< [in] Kernel function pointer to parameterization of cub::InitHistoKernel + MultiBlockHistogramKernelPtr multi_block_kernel, ///< [in] Kernel function pointer to parameterization of cub::MultiBlockHistogramKernel + AggregateHistoKernelPtr aggregate_kernel, ///< [in] Kernel function pointer to parameterization of cub::AggregateHistoKernel + KernelDispachParams &multi_block_dispatch_params, ///< [in] Dispatch parameters that match the policy that \p multi_block_kernel was compiled for + InputIteratorRA d_samples, ///< [in] Input samples to histogram + HistoCounter *d_histograms[ACTIVE_CHANNELS], ///< [out] Array of channel histograms, each having BINS counters of integral type \p HistoCounter. + SizeT num_samples, ///< [in] Number of samples to process + cudaStream_t stream = 0, ///< [in] [optional] CUDA stream to launch kernels within. Default is stream0. + bool stream_synchronous = false) ///< [in] [optional] Whether or not to synchronize the stream after every kernel launch to check for errors. Default is \p false. + { +#ifndef CUB_RUNTIME_ENABLED + + // Kernel launch not supported from this device + return CubDebug(cudaErrorNotSupported); + +#else + + cudaError error = cudaSuccess; + do + { + // Get device ordinal + int device_ordinal; + if (CubDebug(error = cudaGetDevice(&device_ordinal))) break; + + // Get SM count + int sm_count; + if (CubDebug(error = cudaDeviceGetAttribute (&sm_count, cudaDevAttrMultiProcessorCount, device_ordinal))) break; + + // Get a rough estimate of multi_block_kernel SM occupancy based upon the maximum SM occupancy of the targeted PTX architecture + int multi_block_sm_occupancy = CUB_MIN( + ArchProps::MAX_SM_THREADBLOCKS, + ArchProps::MAX_SM_THREADS / multi_block_dispatch_params.block_threads); + +#ifndef __CUDA_ARCH__ + // We're on the host, so come up with a more accurate estimate of multi_block_kernel SM occupancy from actual device properties + Device device_props; + if (CubDebug(error = device_props.Init(device_ordinal))) break; + + if (CubDebug(error = device_props.MaxSmOccupancy( + multi_block_sm_occupancy, + multi_block_kernel, + multi_block_dispatch_params.block_threads))) break; +#endif + + // Get device occupancy for multi_block_kernel + int multi_block_occupancy = multi_block_sm_occupancy * sm_count; + + // Even-share work distribution + GridEvenShare even_share; + + // Get tile size for multi_block_kernel + int multi_block_tile_size = multi_block_dispatch_params.channel_tile_size * CHANNELS; + + // Get grid size for multi_block_kernel + int multi_block_grid_size; + switch (multi_block_dispatch_params.grid_mapping) + { + case GRID_MAPPING_EVEN_SHARE: + + // Work is distributed evenly + even_share.GridInit( + num_samples, + multi_block_occupancy * multi_block_dispatch_params.subscription_factor, + multi_block_tile_size); + multi_block_grid_size = even_share.grid_size; + break; + + case GRID_MAPPING_DYNAMIC: + + // Work is distributed dynamically + int num_tiles = (num_samples + multi_block_tile_size - 1) / multi_block_tile_size; + multi_block_grid_size = (num_tiles < multi_block_occupancy) ? + num_tiles : // Not enough to fill the device with threadblocks + multi_block_occupancy; // Fill the device with threadblocks + break; + }; + + // Temporary storage allocation requirements + void* allocations[2]; + size_t allocation_sizes[2] = + { + ACTIVE_CHANNELS * multi_block_grid_size * sizeof(HistoCounter) * BINS, // bytes needed for privatized histograms + GridQueue::AllocationSize() // bytes needed for grid queue descriptor + }; + + if (CubDebug(error = AliasTemporaries(d_temp_storage, temp_storage_bytes, allocations, allocation_sizes))) break; + + // Return if the caller is simply requesting the size of the storage allocation + if (d_temp_storage == NULL) + return cudaSuccess; + + // Privatized per-block reductions + HistoCounter *d_block_histograms = (HistoCounter*) allocations[0]; + + // Grid queue descriptor + GridQueue queue(allocations[1]); + + // Setup array wrapper for histogram channel output (because we can't pass static arrays as kernel parameters) + ArrayWrapper d_histo_wrapper; + for (int CHANNEL = 0; CHANNEL < ACTIVE_CHANNELS; ++CHANNEL) + d_histo_wrapper.array[CHANNEL] = d_histograms[CHANNEL]; + + // Setup array wrapper for temporary histogram channel output (because we can't pass static arrays as kernel parameters) + ArrayWrapper d_temp_histo_wrapper; + for (int CHANNEL = 0; CHANNEL < ACTIVE_CHANNELS; ++CHANNEL) + d_temp_histo_wrapper.array[CHANNEL] = d_block_histograms + (CHANNEL * multi_block_grid_size * BINS); + + // Log init_kernel configuration + if (stream_synchronous) CubLog("Invoking init_kernel<<<%d, %d, 0, %lld>>>()\n", ACTIVE_CHANNELS, BINS, (long long) stream); + + // Invoke init_kernel to initialize counters and queue descriptor + init_kernel<<>>(queue, d_histo_wrapper, num_samples); + + // Sync the stream if specified + if (stream_synchronous && (CubDebug(error = SyncStream(stream)))) break; + + // Whether we need privatized histograms (i.e., non-global atomics and multi-block) + bool privatized_temporaries = (multi_block_grid_size > 1) && (multi_block_dispatch_params.block_algorithm != GRID_HISTO_GLOBAL_ATOMIC); + + // Log multi_block_kernel configuration + if (stream_synchronous) CubLog("Invoking multi_block_kernel<<<%d, %d, 0, %lld>>>(), %d items per thread, %d SM occupancy\n", + multi_block_grid_size, multi_block_dispatch_params.block_threads, (long long) stream, multi_block_dispatch_params.items_per_thread, multi_block_sm_occupancy); + + // Invoke multi_block_kernel + multi_block_kernel<<>>( + d_samples, + (privatized_temporaries) ? + d_temp_histo_wrapper : + d_histo_wrapper, + num_samples, + even_share, + queue); + + // Sync the stream if specified + if (stream_synchronous && (CubDebug(error = SyncStream(stream)))) break; + + // Aggregate privatized block histograms if necessary + if (privatized_temporaries) + { + // Log aggregate_kernel configuration + if (stream_synchronous) CubLog("Invoking aggregate_kernel<<<%d, %d, 0, %lld>>>()\n", + ACTIVE_CHANNELS, BINS, (long long) stream); + + // Invoke aggregate_kernel + aggregate_kernel<<>>( + d_block_histograms, + d_histo_wrapper, + multi_block_grid_size); + + // Sync the stream if specified + if (stream_synchronous && (CubDebug(error = SyncStream(stream)))) break; + } + } + while (0); + + return error; +#endif // CUB_RUNTIME_ENABLED + } + + + /** + * \brief Computes a device-wide histogram + * + * \tparam GRID_ALGORITHM cub::BlockHistogramTilesAlgorithm enumerator specifying the underlying algorithm to use + * \tparam CHANNELS Number of channels interleaved in the input data (may be greater than the number of channels being actively histogrammed) + * \tparam ACTIVE_CHANNELS [inferred] Number of channels actively being histogrammed + * \tparam InputIteratorRA [inferred] Random-access iterator type for input (may be a simple pointer type) Must have a value type that is assignable to unsigned char + * \tparam HistoCounter [inferred] Integral type for counting sample occurrences per histogram bin + */ + template < + BlockHistogramTilesAlgorithm GRID_ALGORITHM, + int BINS, ///< Number of histogram bins per channel + int CHANNELS, ///< Number of channels interleaved in the input data (may be greater than the number of channels being actively histogrammed) + int ACTIVE_CHANNELS, ///< Number of channels actively being histogrammed + typename InputIteratorRA, ///< The input iterator type (may be a simple pointer type). Must have a value type that is assignable to unsigned char + typename HistoCounter> ///< Integral type for counting sample occurrences per histogram bin + __host__ __device__ __forceinline__ + static cudaError_t Dispatch( + void *d_temp_storage, ///< [in] %Device allocation of temporary storage. When NULL, the required allocation size is returned in \p temp_storage_bytes and no work is done. + size_t &temp_storage_bytes, ///< [in,out] Size in bytes of \p d_temp_storage allocation. + InputIteratorRA d_samples, ///< [in] Input samples to histogram + HistoCounter *d_histograms[ACTIVE_CHANNELS], ///< [out] Array of channel histograms, each having BINS counters of integral type \p HistoCounter. + int num_samples, ///< [in] Number of samples to process + cudaStream_t stream = 0, ///< [in] [optional] CUDA stream to launch kernels within. Default is stream0. + bool stream_synchronous = false) ///< [in] [optional] Whether or not to synchronize the stream after every kernel launch to check for errors. Default is \p false. + { + // Type used for array indexing + typedef int SizeT; + + // Tuning polices for the PTX architecture that will get dispatched to + typedef PtxDefaultPolicies PtxDefaultPolicies; + typedef typename PtxDefaultPolicies::MultiBlockPolicy MultiBlockPolicy; + + cudaError error = cudaSuccess; + do + { + // Declare dispatch parameters + KernelDispachParams multi_block_dispatch_params; + + #ifdef __CUDA_ARCH__ + + // We're on the device, so initialize the dispatch parameters with the PtxDefaultPolicies directly + multi_block_dispatch_params.Init(PtxDefaultPolicies::SUBSCRIPTION_FACTOR); + + #else + + // We're on the host, so lookup and initialize the dispatch parameters with the policies that match the device's PTX version + int ptx_version; + if (CubDebug(error = PtxVersion(ptx_version))) break; + PtxDefaultPolicies::InitDispatchParams(ptx_version, multi_block_dispatch_params); + + #endif + + Dispatch( + d_temp_storage, + temp_storage_bytes, + InitHistoKernel, + MultiBlockHistogramKernel, + AggregateHistoKernel, + multi_block_dispatch_params, + d_samples, + d_histograms, + num_samples, + stream, + stream_synchronous); + + if (CubDebug(error)) break; + } + while (0); + + return error; + } + + #endif // DOXYGEN_SHOULD_SKIP_THIS + + + /******************************************************************//** + * \name Single-channel samples + *********************************************************************/ + //@{ + + + /** + * \brief Computes a device-wide histogram. Uses fast block-sorting to compute the histogram. Delivers consistent throughput regardless of sample diversity, but occupancy may be limited by histogram bin count. + * + * However, because histograms are privatized in shared memory, a large + * number of bins (e.g., thousands) may adversely affect occupancy and + * performance (or even the ability to launch). + * + * \devicestorage + * + * \cdp + * + * \iterator + * + * \par + * The code snippet below illustrates the computation of a 256-bin histogram of + * single-channel unsigned char samples. + * \par + * \code + * #include + * ... + * + * // Declare and initialize device pointers for input samples and 256-bin output histogram + * unsigned char *d_samples; + * unsigned int *d_histogram; + * int num_items = ... + * ... + * + * // Wrap d_samples device pointer in a random-access texture iterator + * cub::TexIteratorRA d_samples_tex_itr; + * d_samples_tex_itr.BindTexture(d_samples, num_items * sizeof(unsigned char)); + * + * // Determine temporary device storage requirements for histogram computation + * void *d_temp_storage = NULL; + * size_t temp_storage_bytes = 0; + * cub::DeviceHistogram::SingleChannelSorting<256>(d_temp_storage, temp_storage_bytes, d_samples_tex_itr, d_histogram, num_items); + * + * // Allocate temporary storage for histogram computation + * cudaMalloc(&d_temp_storage, temp_storage_bytes); + * + * // Compute histogram + * cub::DeviceHistogram::SingleChannelSorting<256>(d_temp_storage, temp_storage_bytes, d_samples_tex_itr, d_histogram, num_items); + * + * // Unbind texture iterator + * d_samples_tex_itr.UnbindTexture(); + * + * \endcode + * + * \tparam BINS Number of histogram bins per channel + * \tparam InputIteratorRA [inferred] Random-access iterator type for input (may be a simple pointer type) Must have a value type that can be cast as an integer in the range [0..BINS-1] + * \tparam HistoCounter [inferred] Integral type for counting sample occurrences per histogram bin + */ + template < + int BINS, + typename InputIteratorRA, + typename HistoCounter> + __host__ __device__ __forceinline__ + static cudaError_t SingleChannelSorting( + void *d_temp_storage, ///< [in] %Device allocation of temporary storage. When NULL, the required allocation size is returned in \p temp_storage_bytes and no work is done. + size_t &temp_storage_bytes, ///< [in,out] Size in bytes of \p d_temp_storage allocation. + InputIteratorRA d_samples, ///< [in] Input samples + HistoCounter* d_histogram, ///< [out] Array of BINS counters of integral type \p HistoCounter. + int num_samples, ///< [in] Number of samples to process + cudaStream_t stream = 0, ///< [in] [optional] CUDA stream to launch kernels within. Default is stream0. + bool stream_synchronous = false) ///< [in] [optional] Whether or not to synchronize the stream after every kernel launch to check for errors. Default is \p false. + { + return Dispatch( + d_temp_storage, temp_storage_bytes, d_samples, &d_histogram, num_samples, stream, stream_synchronous); + } + + + /** + * \brief Computes a device-wide histogram. Uses shared-memory atomic read-modify-write operations to compute the histogram. Input samples having lower diversity can cause performance to be degraded, and occupancy may be limited by histogram bin count. + * + * However, because histograms are privatized in shared memory, a large + * number of bins (e.g., thousands) may adversely affect occupancy and + * performance (or even the ability to launch). + * + * \devicestorage + * + * \cdp + * + * \iterator + * + * \par + * The code snippet below illustrates the computation of a 256-bin histogram of + * single-channel unsigned char samples. + * \par + * \code + * #include + * ... + * + * // Declare and initialize device pointers for input samples and 256-bin output histogram + * unsigned char *d_samples; + * unsigned int *d_histogram; + * int num_items = ... + * ... + * + * // Wrap d_samples device pointer in a random-access texture iterator + * cub::TexIteratorRA d_samples_tex_itr; + * d_samples_tex_itr.BindTexture(d_samples, num_items * sizeof(unsigned char)); + * + * // Determine temporary device storage requirements for histogram computation + * void *d_temp_storage = NULL; + * size_t temp_storage_bytes = 0; + * cub::DeviceHistogram::SingleChannelSorting<256>(d_temp_storage, temp_storage_bytes, d_samples_tex_itr, d_histogram, num_items); + * + * // Allocate temporary storage for histogram computation + * cudaMalloc(&d_temp_storage, temp_storage_bytes); + * + * // Compute histogram + * cub::DeviceHistogram::SingleChannelSharedAtomic<256>(d_temp_storage, temp_storage_bytes, d_samples_tex_itr, d_histogram, num_items); + * + * // Unbind texture iterator + * d_samples_tex_itr.UnbindTexture(); + * + * \endcode + * + * \tparam BINS Number of histogram bins per channel + * \tparam InputIteratorRA [inferred] Random-access iterator type for input (may be a simple pointer type) Must have a value type that can be cast as an integer in the range [0..BINS-1] + * \tparam HistoCounter [inferred] Integral type for counting sample occurrences per histogram bin + */ + template < + int BINS, + typename InputIteratorRA, + typename HistoCounter> + __host__ __device__ __forceinline__ + static cudaError_t SingleChannelSharedAtomic( + void *d_temp_storage, ///< [in] %Device allocation of temporary storage. When NULL, the required allocation size is returned in \p temp_storage_bytes and no work is done. + size_t &temp_storage_bytes, ///< [in,out] Size in bytes of \p d_temp_storage allocation. + InputIteratorRA d_samples, ///< [in] Input samples + HistoCounter* d_histogram, ///< [out] Array of BINS counters of integral type \p HistoCounter. + int num_samples, ///< [in] Number of samples to process + cudaStream_t stream = 0, ///< [in] [optional] CUDA stream to launch kernels within. Default is stream0. + bool stream_synchronous = false) ///< [in] [optional] Whether or not to synchronize the stream after every kernel launch to check for errors. May cause significant slowdown. Default is \p false. + { + return Dispatch( + d_temp_storage, temp_storage_bytes, d_samples, &d_histogram, num_samples, stream, stream_synchronous); + } + + + /** + * \brief Computes a device-wide histogram. Uses global-memory atomic read-modify-write operations to compute the histogram. Input samples having lower diversity can cause performance to be degraded. + * + * Performance is not significantly impacted when computing histograms having large + * numbers of bins (e.g., thousands). + * + * \devicestorage + * + * \cdp + * + * \iterator + * + * \par + * The code snippet below illustrates the computation of a 256-bin histogram of + * single-channel unsigned char samples. + * \par + * \code + * #include + * ... + * + * // Declare and initialize device pointers for input samples and 256-bin output histogram + * unsigned char *d_samples; + * unsigned int *d_histogram; + * int num_items = ... + * ... + * + * // Wrap d_samples device pointer in a random-access texture iterator + * cub::TexIteratorRA d_samples_tex_itr; + * d_samples_tex_itr.BindTexture(d_samples, num_items * sizeof(unsigned char)); + * + * // Determine temporary device storage requirements for histogram computation + * void *d_temp_storage = NULL; + * size_t temp_storage_bytes = 0; + * cub::DeviceHistogram::SingleChannelSorting<256>(d_temp_storage, temp_storage_bytes, d_samples_tex_itr, d_histogram, num_items); + * + * // Allocate temporary storage for histogram computation + * cudaMalloc(&d_temp_storage, temp_storage_bytes); + * + * // Compute histogram + * cub::DeviceHistogram::SingleChannelGlobalAtomic<256>(d_temp_storage, temp_storage_bytes, d_samples_tex_itr, d_histogram, num_items); + * + * // Unbind texture iterator + * d_samples_tex_itr.UnbindTexture(); + * + * \endcode + * + * \tparam BINS Number of histogram bins per channel + * \tparam InputIteratorRA [inferred] Random-access iterator type for input (may be a simple pointer type) Must have a value type that can be cast as an integer in the range [0..BINS-1] + * \tparam HistoCounter [inferred] Integral type for counting sample occurrences per histogram bin + */ + template < + int BINS, + typename InputIteratorRA, + typename HistoCounter> + __host__ __device__ __forceinline__ + static cudaError_t SingleChannelGlobalAtomic( + void *d_temp_storage, ///< [in] %Device allocation of temporary storage. When NULL, the required allocation size is returned in \p temp_storage_bytes and no work is done. + size_t &temp_storage_bytes, ///< [in,out] Size in bytes of \p d_temp_storage allocation. + InputIteratorRA d_samples, ///< [in] Input samples + HistoCounter* d_histogram, ///< [out] Array of BINS counters of integral type \p HistoCounter. + int num_samples, ///< [in] Number of samples to process + cudaStream_t stream = 0, ///< [in] [optional] CUDA stream to launch kernels within. Default is stream0. + bool stream_synchronous = false) ///< [in] [optional] Whether or not to synchronize the stream after every kernel launch to check for errors. May cause significant slowdown. Default is \p false. + { + return Dispatch( + d_temp_storage, temp_storage_bytes, d_samples, &d_histogram, num_samples, stream, stream_synchronous); + } + + + //@} end member group + /******************************************************************//** + * \name Interleaved multi-channel samples + *********************************************************************/ + //@{ + + + /** + * \brief Computes a device-wide histogram from multi-channel data. Uses fast block-sorting to compute the histogram. Delivers consistent throughput regardless of sample diversity, but occupancy may be limited by histogram bin count. + * + * However, because histograms are privatized in shared memory, a large + * number of bins (e.g., thousands) may adversely affect occupancy and + * performance (or even the ability to launch). + * + * The total number of samples across all channels (\p num_samples) must be a whole multiple of \p CHANNELS. + * + * \devicestorage + * + * \cdp + * + * \iterator + * + * \par + * The code snippet below illustrates the computation of three 256-bin histograms from + * interleaved quad-channel unsigned char samples (e.g., RGB histograms from RGBA samples). + * \par + * \code + * #include + * ... + * + * // Declare and initialize device pointers for input samples and + * // three 256-bin output histograms + * unsigned char *d_samples; + * unsigned int *d_histograms[3]; + * int num_items = ... + * ... + * + * // Wrap d_samples device pointer in a random-access texture iterator + * cub::TexIteratorRA d_samples_tex_itr; + * d_samples_tex_itr.BindTexture(d_samples, num_items * sizeof(unsigned char)); + * + * // Determine temporary device storage requirements for histogram computation + * void *d_temp_storage = NULL; + * size_t temp_storage_bytes = 0; + * cub::DeviceHistogram::MultiChannelSorting<256>(d_temp_storage, temp_storage_bytes, d_samples_tex_itr, d_histograms, num_items); + * + * // Allocate temporary storage for histogram computation + * cudaMalloc(&d_temp_storage, temp_storage_bytes); + * + * // Compute histograms + * cub::DeviceHistogram::MultiChannelSorting<256>(d_temp_storage, temp_storage_bytes, d_samples_tex_itr, d_histograms, num_items); + * + * // Unbind texture iterator + * d_samples_tex_itr.UnbindTexture(); + * + * \endcode + * + * \tparam BINS Number of histogram bins per channel + * \tparam CHANNELS Number of channels interleaved in the input data (may be greater than the number of channels being actively histogrammed) + * \tparam ACTIVE_CHANNELS [inferred] Number of channels actively being histogrammed + * \tparam InputIteratorRA [inferred] Random-access iterator type for input (may be a simple pointer type) Must have a value type that can be cast as an integer in the range [0..BINS-1] + * \tparam HistoCounter [inferred] Integral type for counting sample occurrences per histogram bin + */ + template < + int BINS, + int CHANNELS, + int ACTIVE_CHANNELS, + typename InputIteratorRA, + typename HistoCounter> + __host__ __device__ __forceinline__ + static cudaError_t MultiChannelSorting( + void *d_temp_storage, ///< [in] %Device allocation of temporary storage. When NULL, the required allocation size is returned in \p temp_storage_bytes and no work is done. + size_t &temp_storage_bytes, ///< [in,out] Size in bytes of \p d_temp_storage allocation. + InputIteratorRA d_samples, ///< [in] Input samples. The samples from different channels are assumed to be interleaved (e.g., an array of 32b pixels where each pixel consists of four RGBA 8b samples). + HistoCounter *d_histograms[ACTIVE_CHANNELS], ///< [out] Array of channel histogram counter arrays, each having BINS counters of integral type \p HistoCounter. + int num_samples, ///< [in] Total number of samples to process in all channels, including non-active channels + cudaStream_t stream = 0, ///< [in] [optional] CUDA stream to launch kernels within. Default is stream0. + bool stream_synchronous = false) ///< [in] [optional] Whether or not to synchronize the stream after every kernel launch to check for errors. May cause significant slowdown. Default is \p false. + { + return Dispatch( + d_temp_storage, temp_storage_bytes, d_samples, d_histograms, num_samples, stream, stream_synchronous); + } + + + /** + * \brief Computes a device-wide histogram from multi-channel data. Uses shared-memory atomic read-modify-write operations to compute the histogram. Input samples having lower diversity can cause performance to be degraded, and occupancy may be limited by histogram bin count. + * + * However, because histograms are privatized in shared memory, a large + * number of bins (e.g., thousands) may adversely affect occupancy and + * performance (or even the ability to launch). + * + * The total number of samples across all channels (\p num_samples) must be a whole multiple of \p CHANNELS. + * + * \devicestorage + * + * \cdp + * + * \iterator + * + * \par + * The code snippet below illustrates the computation of three 256-bin histograms from + * interleaved quad-channel unsigned char samples (e.g., RGB histograms from RGBA samples). + * \par + * \code + * #include + * ... + * + * // Declare and initialize device pointers for input samples and + * // three 256-bin output histograms + * unsigned char *d_samples; + * unsigned int *d_histograms[3]; + * int num_items = ... + * ... + * + * // Wrap d_samples device pointer in a random-access texture iterator + * cub::TexIteratorRA d_samples_tex_itr; + * d_samples_tex_itr.BindTexture(d_samples, num_items * sizeof(unsigned char)); + * + * // Determine temporary device storage requirements for histogram computation + * void *d_temp_storage = NULL; + * size_t temp_storage_bytes = 0; + * cub::DeviceHistogram::MultiChannelSharedAtomic<256>(d_temp_storage, temp_storage_bytes, d_samples_tex_itr, d_histograms, num_items); + * + * // Allocate temporary storage for histogram computation + * cudaMalloc(&d_temp_storage, temp_storage_bytes); + * + * // Compute histograms + * cub::DeviceHistogram::MultiChannelSharedAtomic<256>(d_temp_storage, temp_storage_bytes, d_samples_tex_itr, d_histograms, num_items); + * + * // Unbind texture iterator + * d_samples_tex_itr.UnbindTexture(); + * + * \endcode + * + * \tparam BINS Number of histogram bins per channel + * \tparam CHANNELS Number of channels interleaved in the input data (may be greater than the number of channels being actively histogrammed) + * \tparam ACTIVE_CHANNELS [inferred] Number of channels actively being histogrammed + * \tparam InputIteratorRA [inferred] Random-access iterator type for input (may be a simple pointer type) Must have a value type that can be cast as an integer in the range [0..BINS-1] + * \tparam HistoCounter [inferred] Integral type for counting sample occurrences per histogram bin + */ + template < + int BINS, + int CHANNELS, + int ACTIVE_CHANNELS, + typename InputIteratorRA, + typename HistoCounter> + __host__ __device__ __forceinline__ + static cudaError_t MultiChannelSharedAtomic( + void *d_temp_storage, ///< [in] %Device allocation of temporary storage. When NULL, the required allocation size is returned in \p temp_storage_bytes and no work is done. + size_t &temp_storage_bytes, ///< [in,out] Size in bytes of \p d_temp_storage allocation. + InputIteratorRA d_samples, ///< [in] Input samples. The samples from different channels are assumed to be interleaved (e.g., an array of 32b pixels where each pixel consists of four RGBA 8b samples). + HistoCounter *d_histograms[ACTIVE_CHANNELS], ///< [out] Array of channel histogram counter arrays, each having BINS counters of integral type \p HistoCounter. + int num_samples, ///< [in] Total number of samples to process in all channels, including non-active channels + cudaStream_t stream = 0, ///< [in] [optional] CUDA stream to launch kernels within. Default is stream0. + bool stream_synchronous = false) ///< [in] [optional] Whether or not to synchronize the stream after every kernel launch to check for errors. May cause significant slowdown. Default is \p false. + { + return Dispatch( + d_temp_storage, temp_storage_bytes, d_samples, d_histograms, num_samples, stream, stream_synchronous); + } + + + /** + * \brief Computes a device-wide histogram from multi-channel data. Uses global-memory atomic read-modify-write operations to compute the histogram. Input samples having lower diversity can cause performance to be degraded. + * + * Performance is not significantly impacted when computing histograms having large + * numbers of bins (e.g., thousands). + * + * The total number of samples across all channels (\p num_samples) must be a whole multiple of \p CHANNELS. + * + * \devicestorage + * + * \cdp + * + * \iterator + * + * Performance is often improved when referencing input samples through a texture-caching iterator, e.g., cub::TexIteratorRA or cub::TexTransformIteratorRA. + * + * \par + * The code snippet below illustrates the computation of three 256-bin histograms from + * interleaved quad-channel unsigned char samples (e.g., RGB histograms from RGBA samples). + * \par + * \code + * #include + * ... + * + * // Declare and initialize device pointers for input samples and + * // three 256-bin output histograms + * unsigned char *d_samples; + * unsigned int *d_histograms[3]; + * int num_items = ... + * ... + * + * // Wrap d_samples device pointer in a random-access texture iterator + * cub::TexIteratorRA d_samples_tex_itr; + * d_samples_tex_itr.BindTexture(d_samples, num_items * sizeof(unsigned char)); + * + * // Determine temporary device storage requirements for histogram computation + * void *d_temp_storage = NULL; + * size_t temp_storage_bytes = 0; + * cub::DeviceHistogram::MultiChannelGlobalAtomic<256>(d_temp_storage, temp_storage_bytes, d_samples_tex_itr, d_histograms, num_items); + * + * // Allocate temporary storage for histogram computation + * cudaMalloc(&d_temp_storage, temp_storage_bytes); + * + * // Compute histograms + * cub::DeviceHistogram::MultiChannelGlobalAtomic<256>(d_temp_storage, temp_storage_bytes, d_samples_tex_itr, d_histograms, num_items); + * + * // Unbind texture iterator + * d_samples_tex_itr.UnbindTexture(); + * + * \endcode + * + * \tparam BINS Number of histogram bins per channel + * \tparam CHANNELS Number of channels interleaved in the input data (may be greater than the number of channels being actively histogrammed) + * \tparam ACTIVE_CHANNELS [inferred] Number of channels actively being histogrammed + * \tparam InputIteratorRA [inferred] Random-access iterator type for input (may be a simple pointer type) Must have a value type that can be cast as an integer in the range [0..BINS-1] + * \tparam HistoCounter [inferred] Integral type for counting sample occurrences per histogram bin + */ + template < + int BINS, + int CHANNELS, + int ACTIVE_CHANNELS, + typename InputIteratorRA, + typename HistoCounter> + __host__ __device__ __forceinline__ + static cudaError_t MultiChannelGlobalAtomic( + void *d_temp_storage, ///< [in] %Device allocation of temporary storage. When NULL, the required allocation size is returned in \p temp_storage_bytes and no work is done. + size_t &temp_storage_bytes, ///< [in,out] Size in bytes of \p d_temp_storage allocation. + InputIteratorRA d_samples, ///< [in] Input samples. The samples from different channels are assumed to be interleaved (e.g., an array of 32b pixels where each pixel consists of four RGBA 8b samples). + HistoCounter *d_histograms[ACTIVE_CHANNELS], ///< [out] Array of channel histogram counter arrays, each having BINS counters of integral type \p HistoCounter. + int num_samples, ///< [in] Total number of samples to process in all channels, including non-active channels + cudaStream_t stream = 0, ///< [in] [optional] CUDA stream to launch kernels within. Default is stream0. + bool stream_synchronous = false) ///< [in] [optional] Whether or not to synchronize the stream after every kernel launch to check for errors. May cause significant slowdown. Default is \p false. + { + return Dispatch( + d_temp_storage, temp_storage_bytes, d_samples, d_histograms, num_samples, stream, stream_synchronous); + } + + //@} end member group + +}; + + +} // CUB namespace +CUB_NS_POSTFIX // Optional outer namespace(s) + + diff --git a/lib/kokkos/TPL/cub/device/device_radix_sort.cuh b/lib/kokkos/TPL/cub/device/device_radix_sort.cuh new file mode 100755 index 0000000000..087d546bc8 --- /dev/null +++ b/lib/kokkos/TPL/cub/device/device_radix_sort.cuh @@ -0,0 +1,890 @@ + +/****************************************************************************** + * Copyright (c) 2011, Duane Merrill. All rights reserved. + * Copyright (c) 2011-2013, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the NVIDIA CORPORATION nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + ******************************************************************************/ + +/** + * \file + * cub::DeviceRadixSort provides operations for computing a device-wide, parallel reduction across data items residing within global memory. + */ + +#pragma once + +#include +#include + +#include "block/block_radix_sort_upsweep_tiles.cuh" +#include "block/block_radix_sort_downsweep_tiles.cuh" +#include "block/block_scan_tiles.cuh" +#include "../grid/grid_even_share.cuh" +#include "../util_debug.cuh" +#include "../util_device.cuh" +#include "../util_namespace.cuh" + +/// Optional outer namespace(s) +CUB_NS_PREFIX + +/// CUB namespace +namespace cub { + +#ifndef DOXYGEN_SHOULD_SKIP_THIS // Do not document + + + + +/****************************************************************************** + * Kernel entry points + *****************************************************************************/ + +/** + * Upsweep pass kernel entry point (multi-block). Computes privatized digit histograms, one per block. + */ +template < + typename BlockRadixSortUpsweepTilesPolicy, ///< Tuning policy for cub::BlockRadixSortUpsweepTiles abstraction + typename Key, ///< Key type + typename SizeT> ///< Integer type used for global array indexing +__launch_bounds__ (int(BlockRadixSortUpsweepTilesPolicy::BLOCK_THREADS), 1) +__global__ void RadixSortUpsweepKernel( + Key *d_keys, ///< [in] Input keys buffer + SizeT *d_spine, ///< [out] Privatized (per block) digit histograms (striped, i.e., 0s counts from each block, then 1s counts from each block, etc.) + SizeT num_items, ///< [in] Total number of input data items + int current_bit, ///< [in] Bit position of current radix digit + bool use_primary_bit_granularity, ///< [in] Whether nor not to use the primary policy (or the embedded alternate policy for smaller bit granularity) + bool first_pass, ///< [in] Whether this is the first digit pass + GridEvenShare even_share) ///< [in] Descriptor for how to map an even-share of tiles across thread blocks +{ + + // Alternate policy for when fewer bits remain + typedef typename BlockRadixSortUpsweepTilesPolicy::AltPolicy AltPolicy; + + // Parameterize two versions of BlockRadixSortUpsweepTiles type for the current configuration + typedef BlockRadixSortUpsweepTiles BlockRadixSortUpsweepTilesT; // Primary + typedef BlockRadixSortUpsweepTiles AltBlockRadixSortUpsweepTilesT; // Alternate (smaller bit granularity) + + // Shared memory storage + __shared__ union + { + typename BlockRadixSortUpsweepTilesT::TempStorage pass_storage; + typename AltBlockRadixSortUpsweepTilesT::TempStorage alt_pass_storage; + } temp_storage; + + // Initialize even-share descriptor for this thread block + even_share.BlockInit(); + + // Process input tiles (each of the first RADIX_DIGITS threads will compute a count for that digit) + if (use_primary_bit_granularity) + { + // Primary granularity + SizeT bin_count; + BlockRadixSortUpsweepTilesT(temp_storage.pass_storage, d_keys, current_bit).ProcessTiles( + even_share.block_offset, + even_share.block_oob, + bin_count); + + // Write out digit counts (striped) + if (threadIdx.x < BlockRadixSortUpsweepTilesT::RADIX_DIGITS) + { + d_spine[(gridDim.x * threadIdx.x) + blockIdx.x] = bin_count; + } + } + else + { + // Alternate granularity + // Process input tiles (each of the first RADIX_DIGITS threads will compute a count for that digit) + SizeT bin_count; + AltBlockRadixSortUpsweepTilesT(temp_storage.alt_pass_storage, d_keys, current_bit).ProcessTiles( + even_share.block_offset, + even_share.block_oob, + bin_count); + + // Write out digit counts (striped) + if (threadIdx.x < AltBlockRadixSortUpsweepTilesT::RADIX_DIGITS) + { + d_spine[(gridDim.x * threadIdx.x) + blockIdx.x] = bin_count; + } + } +} + + +/** + * Spine scan kernel entry point (single-block). Computes an exclusive prefix sum over the privatized digit histograms + */ +template < + typename BlockScanTilesPolicy, ///< Tuning policy for cub::BlockScanTiles abstraction + typename SizeT> ///< Integer type used for global array indexing +__launch_bounds__ (int(BlockScanTilesPolicy::BLOCK_THREADS), 1) +__global__ void RadixSortScanKernel( + SizeT *d_spine, ///< [in,out] Privatized (per block) digit histograms (striped, i.e., 0s counts from each block, then 1s counts from each block, etc.) + int num_counts) ///< [in] Total number of bin-counts +{ + // Parameterize the BlockScanTiles type for the current configuration + typedef BlockScanTiles BlockScanTilesT; + + // Shared memory storage + __shared__ typename BlockScanTilesT::TempStorage temp_storage; + + // Block scan instance + BlockScanTilesT block_scan(temp_storage, d_spine, d_spine, cub::Sum(), SizeT(0)) ; + + // Process full input tiles + int block_offset = 0; + RunningBlockPrefixOp prefix_op; + prefix_op.running_total = 0; + while (block_offset < num_counts) + { + block_scan.ConsumeTile(block_offset, prefix_op); + block_offset += BlockScanTilesT::TILE_ITEMS; + } +} + + +/** + * Downsweep pass kernel entry point (multi-block). Scatters keys (and values) into corresponding bins for the current digit place. + */ +template < + typename BlockRadixSortDownsweepTilesPolicy, ///< Tuning policy for cub::BlockRadixSortUpsweepTiles abstraction + typename Key, ///< Key type + typename Value, ///< Value type + typename SizeT> ///< Integer type used for global array indexing +__launch_bounds__ (int(BlockRadixSortDownsweepTilesPolicy::BLOCK_THREADS)) +__global__ void RadixSortDownsweepKernel( + Key *d_keys_in, ///< [in] Input keys ping buffer + Key *d_keys_out, ///< [in] Output keys pong buffer + Value *d_values_in, ///< [in] Input values ping buffer + Value *d_values_out, ///< [in] Output values pong buffer + SizeT *d_spine, ///< [in] Scan of privatized (per block) digit histograms (striped, i.e., 0s counts from each block, then 1s counts from each block, etc.) + SizeT num_items, ///< [in] Total number of input data items + int current_bit, ///< [in] Bit position of current radix digit + bool use_primary_bit_granularity, ///< [in] Whether nor not to use the primary policy (or the embedded alternate policy for smaller bit granularity) + bool first_pass, ///< [in] Whether this is the first digit pass + bool last_pass, ///< [in] Whether this is the last digit pass + GridEvenShare even_share) ///< [in] Descriptor for how to map an even-share of tiles across thread blocks +{ + + // Alternate policy for when fewer bits remain + typedef typename BlockRadixSortDownsweepTilesPolicy::AltPolicy AltPolicy; + + // Parameterize two versions of BlockRadixSortDownsweepTiles type for the current configuration + typedef BlockRadixSortDownsweepTiles BlockRadixSortDownsweepTilesT; + typedef BlockRadixSortDownsweepTiles AltBlockRadixSortDownsweepTilesT; + + // Shared memory storage + __shared__ union + { + typename BlockRadixSortDownsweepTilesT::TempStorage pass_storage; + typename AltBlockRadixSortDownsweepTilesT::TempStorage alt_pass_storage; + + } temp_storage; + + // Initialize even-share descriptor for this thread block + even_share.BlockInit(); + + if (use_primary_bit_granularity) + { + // Process input tiles + BlockRadixSortDownsweepTilesT(temp_storage.pass_storage, num_items, d_spine, d_keys_in, d_keys_out, d_values_in, d_values_out, current_bit).ProcessTiles( + even_share.block_offset, + even_share.block_oob); + } + else + { + // Process input tiles + AltBlockRadixSortDownsweepTilesT(temp_storage.alt_pass_storage, num_items, d_spine, d_keys_in, d_keys_out, d_values_in, d_values_out, current_bit).ProcessTiles( + even_share.block_offset, + even_share.block_oob); + } +} + + +#endif // DOXYGEN_SHOULD_SKIP_THIS + + + + + +/****************************************************************************** + * DeviceRadixSort + *****************************************************************************/ + +/** + * \brief DeviceRadixSort provides operations for computing a device-wide, parallel radix sort across data items residing within global memory. ![](sorting_logo.png) + * \ingroup DeviceModule + * + * \par Overview + * The [radix sorting method](http://en.wikipedia.org/wiki/Radix_sort) arranges + * items into ascending order. It relies upon a positional representation for + * keys, i.e., each key is comprised of an ordered sequence of symbols (e.g., digits, + * characters, etc.) specified from least-significant to most-significant. For a + * given input sequence of keys and a set of rules specifying a total ordering + * of the symbolic alphabet, the radix sorting method produces a lexicographic + * ordering of those keys. + * + * \par + * DeviceRadixSort can sort all of the built-in C++ numeric primitive types, e.g.: + * unsigned char, \p int, \p double, etc. Although the direct radix sorting + * method can only be applied to unsigned integral types, BlockRadixSort + * is able to sort signed and floating-point types via simple bit-wise transformations + * that ensure lexicographic key ordering. + * + * \par Usage Considerations + * \cdp_class{DeviceRadixSort} + * + * \par Performance + * + * \image html lsd_sort_perf.png + * + */ +struct DeviceRadixSort +{ + #ifndef DOXYGEN_SHOULD_SKIP_THIS // Do not document + + + /****************************************************************************** + * Constants and typedefs + ******************************************************************************/ + + /// Generic structure for encapsulating dispatch properties codified in block policy. + struct KernelDispachParams + { + int block_threads; + int items_per_thread; + cudaSharedMemConfig smem_config; + int radix_bits; + int alt_radix_bits; + int subscription_factor; + int tile_size; + + template + __host__ __device__ __forceinline__ + void InitUpsweepPolicy(int subscription_factor = 1) + { + block_threads = SortBlockPolicy::BLOCK_THREADS; + items_per_thread = SortBlockPolicy::ITEMS_PER_THREAD; + radix_bits = SortBlockPolicy::RADIX_BITS; + alt_radix_bits = SortBlockPolicy::AltPolicy::RADIX_BITS; + smem_config = cudaSharedMemBankSizeFourByte; + this->subscription_factor = subscription_factor; + tile_size = block_threads * items_per_thread; + } + + template + __host__ __device__ __forceinline__ + void InitScanPolicy() + { + block_threads = ScanBlockPolicy::BLOCK_THREADS; + items_per_thread = ScanBlockPolicy::ITEMS_PER_THREAD; + radix_bits = 0; + alt_radix_bits = 0; + smem_config = cudaSharedMemBankSizeFourByte; + subscription_factor = 0; + tile_size = block_threads * items_per_thread; + } + + template + __host__ __device__ __forceinline__ + void InitDownsweepPolicy(int subscription_factor = 1) + { + block_threads = SortBlockPolicy::BLOCK_THREADS; + items_per_thread = SortBlockPolicy::ITEMS_PER_THREAD; + radix_bits = SortBlockPolicy::RADIX_BITS; + alt_radix_bits = SortBlockPolicy::AltPolicy::RADIX_BITS; + smem_config = SortBlockPolicy::SMEM_CONFIG; + this->subscription_factor = subscription_factor; + tile_size = block_threads * items_per_thread; + } + }; + + + + /****************************************************************************** + * Tuning policies + ******************************************************************************/ + + /// Specializations of tuned policy types for different PTX architectures + template + struct TunedPolicies; + + /// SM35 tune + template + struct TunedPolicies + { + enum { + KEYS_ONLY = (Equals::VALUE), + SCALE_FACTOR = (CUB_MAX(sizeof(Key), sizeof(Value)) + 3) / 4, + RADIX_BITS = 5, + }; + + // UpsweepPolicy + typedef BlockRadixSortUpsweepTilesPolicy <64, CUB_MAX(1, 18 / SCALE_FACTOR), LOAD_LDG, RADIX_BITS> UpsweepPolicyKeys; + typedef BlockRadixSortUpsweepTilesPolicy <128, CUB_MAX(1, 15 / SCALE_FACTOR), LOAD_LDG, RADIX_BITS> UpsweepPolicyPairs; + typedef typename If::Type UpsweepPolicy; +/* + // 4bit + typedef BlockRadixSortUpsweepTilesPolicy <128, 15, LOAD_LDG, RADIX_BITS> UpsweepPolicyKeys; + typedef BlockRadixSortUpsweepTilesPolicy <256, 13, LOAD_LDG, RADIX_BITS> UpsweepPolicyPairs; +*/ + // ScanPolicy + typedef BlockScanTilesPolicy <1024, 4, BLOCK_LOAD_VECTORIZE, false, LOAD_DEFAULT, BLOCK_STORE_VECTORIZE, false, BLOCK_SCAN_RAKING_MEMOIZE> ScanPolicy; + + // DownsweepPolicy + typedef BlockRadixSortDownsweepTilesPolicy <64, CUB_MAX(1, 18 / SCALE_FACTOR), BLOCK_LOAD_DIRECT, LOAD_LDG, false, true, BLOCK_SCAN_WARP_SCANS, RADIX_SORT_SCATTER_TWO_PHASE, cudaSharedMemBankSizeEightByte, RADIX_BITS> DownsweepPolicyKeys; + typedef BlockRadixSortDownsweepTilesPolicy <128, CUB_MAX(1, 15 / SCALE_FACTOR), BLOCK_LOAD_DIRECT, LOAD_LDG, false, true, BLOCK_SCAN_WARP_SCANS, RADIX_SORT_SCATTER_TWO_PHASE, cudaSharedMemBankSizeEightByte, RADIX_BITS> DownsweepPolicyPairs; + typedef typename If::Type DownsweepPolicy; + +/* + // 4bit + typedef BlockRadixSortDownsweepTilesPolicy <128, 15, BLOCK_LOAD_DIRECT, LOAD_LDG, false, true, BLOCK_SCAN_WARP_SCANS, RADIX_SORT_SCATTER_TWO_PHASE, cudaSharedMemBankSizeEightByte, RADIX_BITS> DownsweepPolicyKeys; + typedef BlockRadixSortDownsweepTilesPolicy <256, 13, BLOCK_LOAD_DIRECT, LOAD_LDG, false, true, BLOCK_SCAN_WARP_SCANS, RADIX_SORT_SCATTER_TWO_PHASE, cudaSharedMemBankSizeEightByte, RADIX_BITS> DownsweepPolicyPairs; +*/ + enum { SUBSCRIPTION_FACTOR = 7 }; + }; + + + /// SM20 tune + template + struct TunedPolicies + { + enum { + KEYS_ONLY = (Equals::VALUE), + SCALE_FACTOR = (CUB_MAX(sizeof(Key), sizeof(Value)) + 3) / 4, + RADIX_BITS = 5, + }; + + // UpsweepPolicy + typedef BlockRadixSortUpsweepTilesPolicy <64, CUB_MAX(1, 18 / SCALE_FACTOR), LOAD_DEFAULT, RADIX_BITS> UpsweepPolicyKeys; + typedef BlockRadixSortUpsweepTilesPolicy <128, CUB_MAX(1, 13 / SCALE_FACTOR), LOAD_DEFAULT, RADIX_BITS> UpsweepPolicyPairs; + typedef typename If::Type UpsweepPolicy; + + // ScanPolicy + typedef BlockScanTilesPolicy <512, 4, BLOCK_LOAD_VECTORIZE, false, LOAD_DEFAULT, BLOCK_STORE_VECTORIZE, false, BLOCK_SCAN_RAKING_MEMOIZE> ScanPolicy; + + // DownsweepPolicy + typedef BlockRadixSortDownsweepTilesPolicy <64, CUB_MAX(1, 18 / SCALE_FACTOR), BLOCK_LOAD_WARP_TRANSPOSE, LOAD_DEFAULT, false, false, BLOCK_SCAN_WARP_SCANS, RADIX_SORT_SCATTER_TWO_PHASE, cudaSharedMemBankSizeFourByte, RADIX_BITS> DownsweepPolicyKeys; + typedef BlockRadixSortDownsweepTilesPolicy <128, CUB_MAX(1, 13 / SCALE_FACTOR), BLOCK_LOAD_WARP_TRANSPOSE, LOAD_DEFAULT, false, false, BLOCK_SCAN_WARP_SCANS, RADIX_SORT_SCATTER_TWO_PHASE, cudaSharedMemBankSizeFourByte, RADIX_BITS> DownsweepPolicyPairs; + typedef typename If::Type DownsweepPolicy; + + enum { SUBSCRIPTION_FACTOR = 3 }; + }; + + + /// SM10 tune + template + struct TunedPolicies + { + enum { + RADIX_BITS = 4, + }; + + // UpsweepPolicy + typedef BlockRadixSortUpsweepTilesPolicy <64, 9, LOAD_DEFAULT, RADIX_BITS> UpsweepPolicy; + + // ScanPolicy + typedef BlockScanTilesPolicy <256, 4, BLOCK_LOAD_VECTORIZE, false, LOAD_DEFAULT, BLOCK_STORE_VECTORIZE, false, BLOCK_SCAN_RAKING_MEMOIZE> ScanPolicy; + + // DownsweepPolicy + typedef BlockRadixSortDownsweepTilesPolicy <64, 9, BLOCK_LOAD_WARP_TRANSPOSE, LOAD_DEFAULT, false, false, BLOCK_SCAN_WARP_SCANS, RADIX_SORT_SCATTER_TWO_PHASE, cudaSharedMemBankSizeFourByte, RADIX_BITS> DownsweepPolicy; + + enum { SUBSCRIPTION_FACTOR = 3 }; + }; + + + + /****************************************************************************** + * Default policy initializer + ******************************************************************************/ + + /// Tuning policy for the PTX architecture that DeviceRadixSort operations will get dispatched to + template + struct PtxDefaultPolicies + { + + static const int PTX_TUNE_ARCH = (CUB_PTX_ARCH >= 350) ? + 350 : + (CUB_PTX_ARCH >= 200) ? + 200 : + 100; + + // Tuned policy set for the current PTX compiler pass + typedef TunedPolicies PtxTunedPolicies; + + // UpsweepPolicy that opaquely derives from the specialization corresponding to the current PTX compiler pass + struct UpsweepPolicy : PtxTunedPolicies::UpsweepPolicy {}; + + // ScanPolicy that opaquely derives from the specialization corresponding to the current PTX compiler pass + struct ScanPolicy : PtxTunedPolicies::ScanPolicy {}; + + // DownsweepPolicy that opaquely derives from the specialization corresponding to the current PTX compiler pass + struct DownsweepPolicy : PtxTunedPolicies::DownsweepPolicy {}; + + // Subscription factor for the current PTX compiler pass + enum { SUBSCRIPTION_FACTOR = PtxTunedPolicies::SUBSCRIPTION_FACTOR }; + + + /** + * Initialize dispatch params with the policies corresponding to the PTX assembly we will use + */ + static void InitDispatchParams( + int ptx_version, + KernelDispachParams &upsweep_dispatch_params, + KernelDispachParams &scan_dispatch_params, + KernelDispachParams &downsweep_dispatch_params) + { + if (ptx_version >= 350) + { + typedef TunedPolicies TunedPolicies; + upsweep_dispatch_params.InitUpsweepPolicy(TunedPolicies::SUBSCRIPTION_FACTOR); + scan_dispatch_params.InitScanPolicy(); + downsweep_dispatch_params.InitDownsweepPolicy(TunedPolicies::SUBSCRIPTION_FACTOR); + } + else if (ptx_version >= 200) + { + typedef TunedPolicies TunedPolicies; + upsweep_dispatch_params.InitUpsweepPolicy(TunedPolicies::SUBSCRIPTION_FACTOR); + scan_dispatch_params.InitScanPolicy(); + downsweep_dispatch_params.InitDownsweepPolicy(TunedPolicies::SUBSCRIPTION_FACTOR); + } + else + { + typedef TunedPolicies TunedPolicies; + upsweep_dispatch_params.InitUpsweepPolicy(TunedPolicies::SUBSCRIPTION_FACTOR); + scan_dispatch_params.InitScanPolicy(); + downsweep_dispatch_params.InitDownsweepPolicy(TunedPolicies::SUBSCRIPTION_FACTOR); + } + } + }; + + + + /****************************************************************************** + * Utility methods + ******************************************************************************/ + + /** + * Internal dispatch routine for computing a device-wide reduction using a two-stages of kernel invocations. + */ + template < + typename UpsweepKernelPtr, ///< Function type of cub::RadixSortUpsweepKernel + typename SpineKernelPtr, ///< Function type of cub::SpineScanKernel + typename DownsweepKernelPtr, ///< Function type of cub::RadixSortUpsweepKernel + typename Key, ///< Key type + typename Value, ///< Value type + typename SizeT> ///< Integer type used for global array indexing + __host__ __device__ __forceinline__ + static cudaError_t Dispatch( + void *d_temp_storage, ///< [in] %Device allocation of temporary storage. When NULL, the required allocation size is returned in \p temp_storage_bytes and no work is done. + size_t &temp_storage_bytes, ///< [in,out] Size in bytes of \p d_temp_storage allocation. + UpsweepKernelPtr upsweep_kernel, ///< [in] Kernel function pointer to parameterization of cub::RadixSortUpsweepKernel + SpineKernelPtr scan_kernel, ///< [in] Kernel function pointer to parameterization of cub::SpineScanKernel + DownsweepKernelPtr downsweep_kernel, ///< [in] Kernel function pointer to parameterization of cub::RadixSortUpsweepKernel + KernelDispachParams &upsweep_dispatch_params, ///< [in] Dispatch parameters that match the policy that \p upsweep_kernel was compiled for + KernelDispachParams &scan_dispatch_params, ///< [in] Dispatch parameters that match the policy that \p scan_kernel was compiled for + KernelDispachParams &downsweep_dispatch_params, ///< [in] Dispatch parameters that match the policy that \p downsweep_kernel was compiled for + DoubleBuffer &d_keys, ///< [in,out] Double-buffer whose current buffer contains the unsorted input keys and, upon return, is updated to point to the sorted output keys + DoubleBuffer &d_values, ///< [in,out] Double-buffer whose current buffer contains the unsorted input values and, upon return, is updated to point to the sorted output values + SizeT num_items, ///< [in] Number of items to reduce + int begin_bit = 0, ///< [in] [optional] The beginning (least-significant) bit index needed for key comparison + int end_bit = sizeof(Key) * 8, ///< [in] [optional] The past-the-end (most-significant) bit index needed for key comparison + cudaStream_t stream = 0, ///< [in] [optional] CUDA stream to launch kernels within. Default is stream0. + bool stream_synchronous = false) ///< [in] [optional] Whether or not to synchronize the stream after every kernel launch to check for errors. Default is \p false. + { +#ifndef CUB_RUNTIME_ENABLED + + // Kernel launch not supported from this device + return CubDebug(cudaErrorNotSupported ); + +#else + + cudaError error = cudaSuccess; + do + { + // Get device ordinal + int device_ordinal; + if (CubDebug(error = cudaGetDevice(&device_ordinal))) break; + + // Get SM count + int sm_count; + if (CubDebug(error = cudaDeviceGetAttribute (&sm_count, cudaDevAttrMultiProcessorCount, device_ordinal))) break; + + // Get a rough estimate of downsweep_kernel SM occupancy based upon the maximum SM occupancy of the targeted PTX architecture + int downsweep_sm_occupancy = CUB_MIN( + ArchProps::MAX_SM_THREADBLOCKS, + ArchProps::MAX_SM_THREADS / downsweep_dispatch_params.block_threads); + int upsweep_sm_occupancy = downsweep_sm_occupancy; + +#ifndef __CUDA_ARCH__ + // We're on the host, so come up with more accurate estimates of SM occupancy from actual device properties + Device device_props; + if (CubDebug(error = device_props.Init(device_ordinal))) break; + + if (CubDebug(error = device_props.MaxSmOccupancy( + downsweep_sm_occupancy, + downsweep_kernel, + downsweep_dispatch_params.block_threads))) break; + + if (CubDebug(error = device_props.MaxSmOccupancy( + upsweep_sm_occupancy, + upsweep_kernel, + upsweep_dispatch_params.block_threads))) break; +#endif + // Get device occupancies + int downsweep_occupancy = downsweep_sm_occupancy * sm_count; + + // Get even-share work distribution descriptor + GridEvenShare even_share; + int max_downsweep_grid_size = downsweep_occupancy * downsweep_dispatch_params.subscription_factor; + int downsweep_grid_size; + even_share.GridInit(num_items, max_downsweep_grid_size, downsweep_dispatch_params.tile_size); + downsweep_grid_size = even_share.grid_size; + + // Get number of spine elements (round up to nearest spine scan kernel tile size) + int bins = 1 << downsweep_dispatch_params.radix_bits; + int spine_size = downsweep_grid_size * bins; + int spine_tiles = (spine_size + scan_dispatch_params.tile_size - 1) / scan_dispatch_params.tile_size; + spine_size = spine_tiles * scan_dispatch_params.tile_size; + + int alt_bins = 1 << downsweep_dispatch_params.alt_radix_bits; + int alt_spine_size = downsweep_grid_size * alt_bins; + int alt_spine_tiles = (alt_spine_size + scan_dispatch_params.tile_size - 1) / scan_dispatch_params.tile_size; + alt_spine_size = alt_spine_tiles * scan_dispatch_params.tile_size; + + // Temporary storage allocation requirements + void* allocations[1]; + size_t allocation_sizes[1] = + { + spine_size * sizeof(SizeT), // bytes needed for privatized block digit histograms + }; + + // Alias temporaries (or set the necessary size of the storage allocation) + if (CubDebug(error = AliasTemporaries(d_temp_storage, temp_storage_bytes, allocations, allocation_sizes))) break; + + // Return if the caller is simply requesting the size of the storage allocation + if (d_temp_storage == NULL) + return cudaSuccess; + + // Privatized per-block digit histograms + SizeT *d_spine = (SizeT*) allocations[0]; + +#ifndef __CUDA_ARCH__ + // Get current smem bank configuration + cudaSharedMemConfig original_smem_config; + if (CubDebug(error = cudaDeviceGetSharedMemConfig(&original_smem_config))) break; + cudaSharedMemConfig current_smem_config = original_smem_config; +#endif + // Iterate over digit places + int current_bit = begin_bit; + while (current_bit < end_bit) + { + // Use primary bit granularity if bits remaining is a whole multiple of bit primary granularity + int bits_remaining = end_bit - current_bit; + bool use_primary_bit_granularity = (bits_remaining % downsweep_dispatch_params.radix_bits == 0); + int radix_bits = (use_primary_bit_granularity) ? + downsweep_dispatch_params.radix_bits : + downsweep_dispatch_params.alt_radix_bits; + +#ifndef __CUDA_ARCH__ + // Update smem config if necessary + if (current_smem_config != upsweep_dispatch_params.smem_config) + { + if (CubDebug(error = cudaDeviceSetSharedMemConfig(upsweep_dispatch_params.smem_config))) break; + current_smem_config = upsweep_dispatch_params.smem_config; + } +#endif + + // Log upsweep_kernel configuration + if (stream_synchronous) + CubLog("Invoking upsweep_kernel<<<%d, %d, 0, %lld>>>(), %d smem config, %d items per thread, %d SM occupancy, selector %d, current bit %d, bit_grain %d\n", + downsweep_grid_size, upsweep_dispatch_params.block_threads, (long long) stream, upsweep_dispatch_params.smem_config, upsweep_dispatch_params.items_per_thread, upsweep_sm_occupancy, d_keys.selector, current_bit, radix_bits); + + // Invoke upsweep_kernel with same grid size as downsweep_kernel + upsweep_kernel<<>>( + d_keys.d_buffers[d_keys.selector], + d_spine, + num_items, + current_bit, + use_primary_bit_granularity, + (current_bit == begin_bit), + even_share); + + // Sync the stream if specified + if (stream_synchronous && (CubDebug(error = SyncStream(stream)))) break; + + // Log scan_kernel configuration + if (stream_synchronous) CubLog("Invoking scan_kernel<<<%d, %d, 0, %lld>>>(), %d items per thread\n", + 1, scan_dispatch_params.block_threads, (long long) stream, scan_dispatch_params.items_per_thread); + + // Invoke scan_kernel + scan_kernel<<<1, scan_dispatch_params.block_threads, 0, stream>>>( + d_spine, + (use_primary_bit_granularity) ? spine_size : alt_spine_size); + + // Sync the stream if specified + if (stream_synchronous && (CubDebug(error = SyncStream(stream)))) break; + +#ifndef __CUDA_ARCH__ + // Update smem config if necessary + if (current_smem_config != downsweep_dispatch_params.smem_config) + { + if (CubDebug(error = cudaDeviceSetSharedMemConfig(downsweep_dispatch_params.smem_config))) break; + current_smem_config = downsweep_dispatch_params.smem_config; + } +#endif + + // Log downsweep_kernel configuration + if (stream_synchronous) CubLog("Invoking downsweep_kernel<<<%d, %d, 0, %lld>>>(), %d smem config, %d items per thread, %d SM occupancy\n", + downsweep_grid_size, downsweep_dispatch_params.block_threads, (long long) stream, downsweep_dispatch_params.smem_config, downsweep_dispatch_params.items_per_thread, downsweep_sm_occupancy); + + // Invoke downsweep_kernel + downsweep_kernel<<>>( + d_keys.d_buffers[d_keys.selector], + d_keys.d_buffers[d_keys.selector ^ 1], + d_values.d_buffers[d_values.selector], + d_values.d_buffers[d_values.selector ^ 1], + d_spine, + num_items, + current_bit, + use_primary_bit_granularity, + (current_bit == begin_bit), + (current_bit + downsweep_dispatch_params.radix_bits >= end_bit), + even_share); + + // Sync the stream if specified + if (stream_synchronous && (CubDebug(error = SyncStream(stream)))) break; + + // Invert selectors + d_keys.selector ^= 1; + d_values.selector ^= 1; + + // Update current bit position + current_bit += radix_bits; + } + +#ifndef __CUDA_ARCH__ + // Reset smem config if necessary + if (current_smem_config != original_smem_config) + { + if (CubDebug(error = cudaDeviceSetSharedMemConfig(original_smem_config))) break; + } +#endif + + } + while (0); + + return error; + +#endif // CUB_RUNTIME_ENABLED + } + + + + #endif // DOXYGEN_SHOULD_SKIP_THIS + + /****************************************************************************** + * Interface + ******************************************************************************/ + + + /** + * \brief Sorts key-value pairs. + * + * \par + * The sorting operation requires a pair of key buffers and a pair of value + * buffers. Each pair is wrapped in a DoubleBuffer structure whose member + * DoubleBuffer::Current() references the active buffer. The currently-active + * buffer may be changed by the sorting operation. + * + * \devicestorage + * + * \cdp + * + * \par + * The code snippet below illustrates the sorting of a device vector of \p int keys + * with associated vector of \p int values. + * \par + * \code + * #include + * ... + * + * // Create a set of DoubleBuffers to wrap pairs of device pointers for + * // sorting data (keys, values, and equivalently-sized alternate buffers) + * int num_items = ... + * cub::DoubleBuffer d_keys(d_key_buf, d_key_alt_buf); + * cub::DoubleBuffer d_values(d_value_buf, d_value_alt_buf); + * + * // Determine temporary device storage requirements for sorting operation + * void *d_temp_storage = NULL; + * size_t temp_storage_bytes = 0; + * cub::DeviceRadixSort::SortKeys(d_temp_storage, temp_storage_bytes, d_keys, num_items); + * + * // Allocate temporary storage for sorting operation + * cudaMalloc(&d_temp_storage, temp_storage_bytes); + * + * // Run sorting operation + * cub::DeviceRadixSort::SortKeys(d_temp_storage, temp_storage_bytes, d_keys, num_items); + * + * // Sorted keys and values are referenced by d_keys.Current() and d_values.Current() + * + * \endcode + * + * \tparam Key [inferred] Key type + * \tparam Value [inferred] Value type + */ + template < + typename Key, + typename Value> + __host__ __device__ __forceinline__ + static cudaError_t SortPairs( + void *d_temp_storage, ///< [in] %Device allocation of temporary storage. When NULL, the required allocation size is returned in \p temp_storage_bytes and no work is done. + size_t &temp_storage_bytes, ///< [in,out] Size in bytes of \p d_temp_storage allocation. + DoubleBuffer &d_keys, ///< [in,out] Double-buffer of keys whose current buffer contains the unsorted input keys and, upon return, is updated to point to the sorted output keys + DoubleBuffer &d_values, ///< [in,out] Double-buffer of values whose current buffer contains the unsorted input values and, upon return, is updated to point to the sorted output values + int num_items, ///< [in] Number of items to reduce + int begin_bit = 0, ///< [in] [optional] The first (least-significant) bit index needed for key comparison + int end_bit = sizeof(Key) * 8, ///< [in] [optional] The past-the-end (most-significant) bit index needed for key comparison + cudaStream_t stream = 0, ///< [in] [optional] CUDA stream to launch kernels within. Default is stream0. + bool stream_synchronous = false) ///< [in] [optional] Whether or not to synchronize the stream after every kernel launch to check for errors. Default is \p false. + { + // Type used for array indexing + typedef int SizeT; + + // Tuning polices + typedef PtxDefaultPolicies PtxDefaultPolicies; // Wrapper of default kernel policies + typedef typename PtxDefaultPolicies::UpsweepPolicy UpsweepPolicy; // Upsweep kernel policy + typedef typename PtxDefaultPolicies::ScanPolicy ScanPolicy; // Scan kernel policy + typedef typename PtxDefaultPolicies::DownsweepPolicy DownsweepPolicy; // Downsweep kernel policy + + cudaError error = cudaSuccess; + do + { + // Declare dispatch parameters + KernelDispachParams upsweep_dispatch_params; + KernelDispachParams scan_dispatch_params; + KernelDispachParams downsweep_dispatch_params; + +#ifdef __CUDA_ARCH__ + // We're on the device, so initialize the dispatch parameters with the PtxDefaultPolicies directly + upsweep_dispatch_params.InitUpsweepPolicy(PtxDefaultPolicies::SUBSCRIPTION_FACTOR); + scan_dispatch_params.InitScanPolicy(); + downsweep_dispatch_params.InitDownsweepPolicy(PtxDefaultPolicies::SUBSCRIPTION_FACTOR); +#else + // We're on the host, so lookup and initialize the dispatch parameters with the policies that match the device's PTX version + int ptx_version; + if (CubDebug(error = PtxVersion(ptx_version))) break; + PtxDefaultPolicies::InitDispatchParams( + ptx_version, + upsweep_dispatch_params, + scan_dispatch_params, + downsweep_dispatch_params); +#endif + // Dispatch + if (CubDebug(error = Dispatch( + d_temp_storage, + temp_storage_bytes, + RadixSortUpsweepKernel, + RadixSortScanKernel, + RadixSortDownsweepKernel, + upsweep_dispatch_params, + scan_dispatch_params, + downsweep_dispatch_params, + d_keys, + d_values, + num_items, + begin_bit, + end_bit, + stream, + stream_synchronous))) break; + } + while (0); + + return error; + } + + + /** + * \brief Sorts keys + * + * \par + * The sorting operation requires a pair of key buffers. The pair is + * wrapped in a DoubleBuffer structure whose member DoubleBuffer::Current() + * references the active buffer. The currently-active buffer may be changed + * by the sorting operation. + * + * \devicestorage + * + * \cdp + * + * \par + * The code snippet below illustrates the sorting of a device vector of \p int keys. + * \par + * \code + * #include + * ... + * + * // Create a set of DoubleBuffers to wrap pairs of device pointers for + * // sorting data (keys and equivalently-sized alternate buffer) + * int num_items = ... + * cub::DoubleBuffer d_keys(d_key_buf, d_key_alt_buf); + * + * // Determine temporary device storage requirements for sorting operation + * void *d_temp_storage = NULL; + * size_t temp_storage_bytes = 0; + * cub::DeviceRadixSort::SortKeys(d_temp_storage, temp_storage_bytes, d_keys, num_items); + * + * // Allocate temporary storage for sorting operation + * cudaMalloc(&d_temp_storage, temp_storage_bytes); + * + * // Run sorting operation + * cub::DeviceRadixSort::SortKeys(d_temp_storage, temp_storage_bytes, d_keys, num_items); + * + * // Sorted keys are referenced by d_keys.Current() + * + * \endcode + * + * \tparam Key [inferred] Key type + */ + template + __host__ __device__ __forceinline__ + static cudaError_t SortKeys( + void *d_temp_storage, ///< [in] %Device allocation of temporary storage. When NULL, the required allocation size is returned in \p temp_storage_bytes and no work is done. + size_t &temp_storage_bytes, ///< [in,out] Size in bytes of \p d_temp_storage allocation. + DoubleBuffer &d_keys, ///< [in,out] Double-buffer of keys whose current buffer contains the unsorted input keys and, upon return, is updated to point to the sorted output keys + int num_items, ///< [in] Number of items to reduce + int begin_bit = 0, ///< [in] [optional] The first (least-significant) bit index needed for key comparison + int end_bit = sizeof(Key) * 8, ///< [in] [optional] The past-the-end (most-significant) bit index needed for key comparison + cudaStream_t stream = 0, ///< [in] [optional] CUDA stream to launch kernels within. Default is stream0. + bool stream_synchronous = false) ///< [in] [optional] Whether or not to synchronize the stream after every kernel launch to check for errors. Default is \p false. + { + DoubleBuffer d_values; + return SortPairs(d_temp_storage, temp_storage_bytes, d_keys, d_values, num_items, begin_bit, end_bit, stream, stream_synchronous); + } + +}; + + +} // CUB namespace +CUB_NS_POSTFIX // Optional outer namespace(s) + + diff --git a/lib/kokkos/TPL/cub/device/device_reduce.cuh b/lib/kokkos/TPL/cub/device/device_reduce.cuh new file mode 100755 index 0000000000..069af8c1fb --- /dev/null +++ b/lib/kokkos/TPL/cub/device/device_reduce.cuh @@ -0,0 +1,775 @@ + +/****************************************************************************** + * Copyright (c) 2011, Duane Merrill. All rights reserved. + * Copyright (c) 2011-2013, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the NVIDIA CORPORATION nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + ******************************************************************************/ + +/** + * \file + * cub::DeviceReduce provides operations for computing a device-wide, parallel reduction across data items residing within global memory. + */ + +#pragma once + +#include +#include + +#include "block/block_reduce_tiles.cuh" +#include "../thread/thread_operators.cuh" +#include "../grid/grid_even_share.cuh" +#include "../grid/grid_queue.cuh" +#include "../util_debug.cuh" +#include "../util_device.cuh" +#include "../util_namespace.cuh" + +/// Optional outer namespace(s) +CUB_NS_PREFIX + +/// CUB namespace +namespace cub { + +#ifndef DOXYGEN_SHOULD_SKIP_THIS // Do not document + + + + + + +/****************************************************************************** + * Kernel entry points + *****************************************************************************/ + +/** + * Reduction pass kernel entry point (multi-block). Computes privatized reductions, one per thread block. + */ +template < + typename BlockReduceTilesPolicy, ///< Tuning policy for cub::BlockReduceTiles abstraction + typename InputIteratorRA, ///< Random-access iterator type for input (may be a simple pointer type) + typename OutputIteratorRA, ///< Random-access iterator type for output (may be a simple pointer type) + typename SizeT, ///< Integer type used for global array indexing + typename ReductionOp> ///< Binary reduction operator type having member T operator()(const T &a, const T &b) +__launch_bounds__ (int(BlockReduceTilesPolicy::BLOCK_THREADS), 1) +__global__ void ReducePrivatizedKernel( + InputIteratorRA d_in, ///< [in] Input data to reduce + OutputIteratorRA d_out, ///< [out] Output location for result + SizeT num_items, ///< [in] Total number of input data items + GridEvenShare even_share, ///< [in] Descriptor for how to map an even-share of tiles across thread blocks + GridQueue queue, ///< [in] Descriptor for performing dynamic mapping of tile data to thread blocks + ReductionOp reduction_op) ///< [in] Binary reduction operator +{ + // Data type + typedef typename std::iterator_traits::value_type T; + + // Thread block type for reducing input tiles + typedef BlockReduceTiles BlockReduceTilesT; + + // Block-wide aggregate + T block_aggregate; + + // Shared memory storage + __shared__ typename BlockReduceTilesT::TempStorage temp_storage; + + // Consume input tiles + BlockReduceTilesT(temp_storage, d_in, reduction_op).ConsumeTiles( + num_items, + even_share, + queue, + block_aggregate, + Int2Type()); + + // Output result + if (threadIdx.x == 0) + { + d_out[blockIdx.x] = block_aggregate; + } +} + + +/** + * Reduction pass kernel entry point (single-block). Aggregates privatized threadblock reductions from a previous multi-block reduction pass. + */ +template < + typename BlockReduceTilesPolicy, ///< Tuning policy for cub::BlockReduceTiles abstraction + typename InputIteratorRA, ///< Random-access iterator type for input (may be a simple pointer type) + typename OutputIteratorRA, ///< Random-access iterator type for output (may be a simple pointer type) + typename SizeT, ///< Integer type used for global array indexing + typename ReductionOp> ///< Binary reduction operator type having member T operator()(const T &a, const T &b) +__launch_bounds__ (int(BlockReduceTilesPolicy::BLOCK_THREADS), 1) +__global__ void ReduceSingleKernel( + InputIteratorRA d_in, ///< [in] Input data to reduce + OutputIteratorRA d_out, ///< [out] Output location for result + SizeT num_items, ///< [in] Total number of input data items + ReductionOp reduction_op) ///< [in] Binary reduction operator +{ + // Data type + typedef typename std::iterator_traits::value_type T; + + // Thread block type for reducing input tiles + typedef BlockReduceTiles BlockReduceTilesT; + + // Block-wide aggregate + T block_aggregate; + + // Shared memory storage + __shared__ typename BlockReduceTilesT::TempStorage temp_storage; + + // Consume input tiles + BlockReduceTilesT(temp_storage, d_in, reduction_op).ConsumeTiles( + SizeT(0), + SizeT(num_items), + block_aggregate); + + // Output result + if (threadIdx.x == 0) + { + d_out[blockIdx.x] = block_aggregate; + } +} + +#endif // DOXYGEN_SHOULD_SKIP_THIS + + +/****************************************************************************** + * DeviceReduce + *****************************************************************************/ + +/** + * \brief DeviceReduce provides operations for computing a device-wide, parallel reduction across data items residing within global memory. ![](reduce_logo.png) + * \ingroup DeviceModule + * + * \par Overview + * A reduction (or fold) + * uses a binary combining operator to compute a single aggregate from a list of input elements. + * + * \par Usage Considerations + * \cdp_class{DeviceReduce} + * + * \par Performance + * + * \image html reduction_perf.png + * + */ +struct DeviceReduce +{ +#ifndef DOXYGEN_SHOULD_SKIP_THIS // Do not document + + + /****************************************************************************** + * Constants and typedefs + ******************************************************************************/ + + /// Generic structure for encapsulating dispatch properties codified in block policy. + struct KernelDispachParams + { + int block_threads; + int items_per_thread; + int vector_load_length; + BlockReduceAlgorithm block_algorithm; + PtxLoadModifier load_modifier; + GridMappingStrategy grid_mapping; + int subscription_factor; + int tile_size; + + template + __host__ __device__ __forceinline__ + void Init(int subscription_factor = 1) + { + block_threads = BlockPolicy::BLOCK_THREADS; + items_per_thread = BlockPolicy::ITEMS_PER_THREAD; + vector_load_length = BlockPolicy::VECTOR_LOAD_LENGTH; + block_algorithm = BlockPolicy::BLOCK_ALGORITHM; + load_modifier = BlockPolicy::LOAD_MODIFIER; + grid_mapping = BlockPolicy::GRID_MAPPING; + this->subscription_factor = subscription_factor; + tile_size = block_threads * items_per_thread; + } + + __host__ __device__ __forceinline__ + void Print() + { + printf("%d threads, %d per thread, %d veclen, %d algo, %d loadmod, %d mapping, %d subscription", + block_threads, + items_per_thread, + vector_load_length, + block_algorithm, + load_modifier, + grid_mapping, + subscription_factor); + } + + }; + + + /****************************************************************************** + * Tuning policies + ******************************************************************************/ + + /// Specializations of tuned policy types for different PTX architectures + template < + typename T, + typename SizeT, + int ARCH> + struct TunedPolicies; + + /// SM35 tune + template + struct TunedPolicies + { + // PrivatizedPolicy (1B): GTX Titan: 206.0 GB/s @ 192M 1B items + typedef BlockReduceTilesPolicy<128, 12, 1, BLOCK_REDUCE_RAKING, LOAD_LDG, GRID_MAPPING_DYNAMIC> PrivatizedPolicy1B; + + // PrivatizedPolicy (4B): GTX Titan: 254.2 GB/s @ 48M 4B items + typedef BlockReduceTilesPolicy<512, 20, 1, BLOCK_REDUCE_RAKING, LOAD_DEFAULT, GRID_MAPPING_EVEN_SHARE> PrivatizedPolicy4B; + + // PrivatizedPolicy + typedef typename If<(sizeof(T) < 4), + PrivatizedPolicy1B, + PrivatizedPolicy4B>::Type PrivatizedPolicy; + + // SinglePolicy + typedef BlockReduceTilesPolicy<256, 8, 1, BLOCK_REDUCE_WARP_REDUCTIONS, LOAD_DEFAULT, GRID_MAPPING_EVEN_SHARE> SinglePolicy; + + enum { SUBSCRIPTION_FACTOR = 7 }; + + }; + + /// SM30 tune + template + struct TunedPolicies + { + // PrivatizedPolicy: GTX670: 154.0 @ 48M 32-bit T + typedef BlockReduceTilesPolicy<256, 2, 1, BLOCK_REDUCE_WARP_REDUCTIONS, LOAD_DEFAULT, GRID_MAPPING_EVEN_SHARE> PrivatizedPolicy; + + // SinglePolicy + typedef BlockReduceTilesPolicy<256, 24, 4, BLOCK_REDUCE_WARP_REDUCTIONS, LOAD_DEFAULT, GRID_MAPPING_EVEN_SHARE> SinglePolicy; + + enum { SUBSCRIPTION_FACTOR = 1 }; + }; + + /// SM20 tune + template + struct TunedPolicies + { + // PrivatizedPolicy (1B): GTX 580: 158.1 GB/s @ 192M 1B items + typedef BlockReduceTilesPolicy<192, 24, 4, BLOCK_REDUCE_RAKING, LOAD_DEFAULT, GRID_MAPPING_EVEN_SHARE> PrivatizedPolicy1B; + + // PrivatizedPolicy (4B): GTX 580: 178.9 GB/s @ 48M 4B items + typedef BlockReduceTilesPolicy<128, 8, 4, BLOCK_REDUCE_RAKING, LOAD_DEFAULT, GRID_MAPPING_DYNAMIC> PrivatizedPolicy4B; + + // PrivatizedPolicy + typedef typename If<(sizeof(T) < 4), + PrivatizedPolicy1B, + PrivatizedPolicy4B>::Type PrivatizedPolicy; + + // SinglePolicy + typedef BlockReduceTilesPolicy<192, 7, 1, BLOCK_REDUCE_RAKING, LOAD_DEFAULT, GRID_MAPPING_EVEN_SHARE> SinglePolicy; + + enum { SUBSCRIPTION_FACTOR = 2 }; + }; + + /// SM13 tune + template + struct TunedPolicies + { + // PrivatizedPolicy + typedef BlockReduceTilesPolicy<128, 8, 2, BLOCK_REDUCE_RAKING, LOAD_DEFAULT, GRID_MAPPING_EVEN_SHARE> PrivatizedPolicy; + + // SinglePolicy + typedef BlockReduceTilesPolicy<32, 4, 4, BLOCK_REDUCE_RAKING, LOAD_DEFAULT, GRID_MAPPING_EVEN_SHARE> SinglePolicy; + + enum { SUBSCRIPTION_FACTOR = 1 }; + }; + + /// SM10 tune + template + struct TunedPolicies + { + // PrivatizedPolicy + typedef BlockReduceTilesPolicy<128, 8, 2, BLOCK_REDUCE_RAKING, LOAD_DEFAULT, GRID_MAPPING_EVEN_SHARE> PrivatizedPolicy; + + // SinglePolicy + typedef BlockReduceTilesPolicy<32, 4, 4, BLOCK_REDUCE_RAKING, LOAD_DEFAULT, GRID_MAPPING_EVEN_SHARE> SinglePolicy; + + enum { SUBSCRIPTION_FACTOR = 1 }; + }; + + + + /****************************************************************************** + * Default policy initializer + ******************************************************************************/ + + /// Tuning policy for the PTX architecture that DeviceReduce operations will get dispatched to + template + struct PtxDefaultPolicies + { + static const int PTX_TUNE_ARCH = (CUB_PTX_ARCH >= 350) ? + 350 : + (CUB_PTX_ARCH >= 300) ? + 300 : + (CUB_PTX_ARCH >= 200) ? + 200 : + (CUB_PTX_ARCH >= 130) ? + 130 : + 100; + + // Tuned policy set for the current PTX compiler pass + typedef TunedPolicies PtxTunedPolicies; + + // Subscription factor for the current PTX compiler pass + static const int SUBSCRIPTION_FACTOR = PtxTunedPolicies::SUBSCRIPTION_FACTOR; + + // PrivatizedPolicy that opaquely derives from the specialization corresponding to the current PTX compiler pass + struct PrivatizedPolicy : PtxTunedPolicies::PrivatizedPolicy {}; + + // SinglePolicy that opaquely derives from the specialization corresponding to the current PTX compiler pass + struct SinglePolicy : PtxTunedPolicies::SinglePolicy {}; + + + /** + * Initialize dispatch params with the policies corresponding to the PTX assembly we will use + */ + static void InitDispatchParams( + int ptx_version, + KernelDispachParams &privatized_dispatch_params, + KernelDispachParams &single_dispatch_params) + { + if (ptx_version >= 350) + { + typedef TunedPolicies TunedPolicies; + privatized_dispatch_params.Init(TunedPolicies::SUBSCRIPTION_FACTOR); + single_dispatch_params.Init(); + } + else if (ptx_version >= 300) + { + typedef TunedPolicies TunedPolicies; + privatized_dispatch_params.Init(TunedPolicies::SUBSCRIPTION_FACTOR); + single_dispatch_params.Init(); + } + else if (ptx_version >= 200) + { + typedef TunedPolicies TunedPolicies; + privatized_dispatch_params.Init(TunedPolicies::SUBSCRIPTION_FACTOR); + single_dispatch_params.Init(); + } + else if (ptx_version >= 130) + { + typedef TunedPolicies TunedPolicies; + privatized_dispatch_params.Init(TunedPolicies::SUBSCRIPTION_FACTOR); + single_dispatch_params.Init(); + } + else + { + typedef TunedPolicies TunedPolicies; + privatized_dispatch_params.Init(TunedPolicies::SUBSCRIPTION_FACTOR); + single_dispatch_params.Init(); + } + } + }; + + + + /****************************************************************************** + * Utility methods + ******************************************************************************/ + + /** + * Internal dispatch routine for computing a device-wide reduction using a two-stages of kernel invocations. + */ + template < + typename ReducePrivatizedKernelPtr, ///< Function type of cub::ReducePrivatizedKernel + typename ReduceSingleKernelPtr, ///< Function type of cub::ReduceSingleKernel + typename ResetDrainKernelPtr, ///< Function type of cub::ResetDrainKernel + typename InputIteratorRA, ///< Random-access iterator type for input (may be a simple pointer type) + typename OutputIteratorRA, ///< Random-access iterator type for output (may be a simple pointer type) + typename SizeT, ///< Integer type used for global array indexing + typename ReductionOp> ///< Binary reduction operator type having member T operator()(const T &a, const T &b) + __host__ __device__ __forceinline__ + static cudaError_t Dispatch( + void *d_temp_storage, ///< [in] %Device allocation of temporary storage. When NULL, the required allocation size is returned in \p temp_storage_bytes and no work is done. + size_t &temp_storage_bytes, ///< [in,out] Size in bytes of \p d_temp_storage allocation. + ReducePrivatizedKernelPtr privatized_kernel, ///< [in] Kernel function pointer to parameterization of cub::ReducePrivatizedKernel + ReduceSingleKernelPtr single_kernel, ///< [in] Kernel function pointer to parameterization of cub::ReduceSingleKernel + ResetDrainKernelPtr prepare_drain_kernel, ///< [in] Kernel function pointer to parameterization of cub::ResetDrainKernel + KernelDispachParams &privatized_dispatch_params, ///< [in] Dispatch parameters that match the policy that \p privatized_kernel_ptr was compiled for + KernelDispachParams &single_dispatch_params, ///< [in] Dispatch parameters that match the policy that \p single_kernel was compiled for + InputIteratorRA d_in, ///< [in] Input data to reduce + OutputIteratorRA d_out, ///< [out] Output location for result + SizeT num_items, ///< [in] Number of items to reduce + ReductionOp reduction_op, ///< [in] Binary reduction operator + cudaStream_t stream = 0, ///< [in] [optional] CUDA stream to launch kernels within. Default is stream0. + bool stream_synchronous = false) ///< [in] [optional] Whether or not to synchronize the stream after every kernel launch to check for errors. Default is \p false. + { +#ifndef CUB_RUNTIME_ENABLED + + // Kernel launch not supported from this device + return CubDebug(cudaErrorNotSupported ); + +#else + + // Data type of input iterator + typedef typename std::iterator_traits::value_type T; + + cudaError error = cudaSuccess; + do + { + if ((privatized_kernel == NULL) || (num_items <= (single_dispatch_params.tile_size))) + { + // Dispatch a single-block reduction kernel + + // Return if the caller is simply requesting the size of the storage allocation + if (d_temp_storage == NULL) + { + temp_storage_bytes = 1; + return cudaSuccess; + } + + // Log single_kernel configuration + if (stream_synchronous) CubLog("Invoking ReduceSingle<<<1, %d, 0, %lld>>>(), %d items per thread\n", + single_dispatch_params.block_threads, (long long) stream, single_dispatch_params.items_per_thread); + + // Invoke single_kernel + single_kernel<<<1, single_dispatch_params.block_threads>>>( + d_in, + d_out, + num_items, + reduction_op); + + // Sync the stream if specified + if (stream_synchronous && (CubDebug(error = SyncStream(stream)))) break; + + } + else + { + // Dispatch two kernels: a multi-block kernel to compute + // privatized per-block reductions, and then a single-block + // to reduce those + + // Get device ordinal + int device_ordinal; + if (CubDebug(error = cudaGetDevice(&device_ordinal))) break; + + // Get SM count + int sm_count; + if (CubDebug(error = cudaDeviceGetAttribute (&sm_count, cudaDevAttrMultiProcessorCount, device_ordinal))) break; + + // Get a rough estimate of privatized_kernel SM occupancy based upon the maximum SM occupancy of the targeted PTX architecture + int privatized_sm_occupancy = CUB_MIN( + ArchProps::MAX_SM_THREADBLOCKS, + ArchProps::MAX_SM_THREADS / privatized_dispatch_params.block_threads); + +#ifndef __CUDA_ARCH__ + // We're on the host, so come up with a more accurate estimate of privatized_kernel SM occupancy from actual device properties + Device device_props; + if (CubDebug(error = device_props.Init(device_ordinal))) break; + + if (CubDebug(error = device_props.MaxSmOccupancy( + privatized_sm_occupancy, + privatized_kernel, + privatized_dispatch_params.block_threads))) break; +#endif + + // Get device occupancy for privatized_kernel + int privatized_occupancy = privatized_sm_occupancy * sm_count; + + // Even-share work distribution + GridEvenShare even_share; + + // Get grid size for privatized_kernel + int privatized_grid_size; + switch (privatized_dispatch_params.grid_mapping) + { + case GRID_MAPPING_EVEN_SHARE: + + // Work is distributed evenly + even_share.GridInit( + num_items, + privatized_occupancy * privatized_dispatch_params.subscription_factor, + privatized_dispatch_params.tile_size); + privatized_grid_size = even_share.grid_size; + break; + + case GRID_MAPPING_DYNAMIC: + + // Work is distributed dynamically + int num_tiles = (num_items + privatized_dispatch_params.tile_size - 1) / privatized_dispatch_params.tile_size; + privatized_grid_size = (num_tiles < privatized_occupancy) ? + num_tiles : // Not enough to fill the device with threadblocks + privatized_occupancy; // Fill the device with threadblocks + break; + }; + + // Temporary storage allocation requirements + void* allocations[2]; + size_t allocation_sizes[2] = + { + privatized_grid_size * sizeof(T), // bytes needed for privatized block reductions + GridQueue::AllocationSize() // bytes needed for grid queue descriptor + }; + + // Alias temporaries (or set the necessary size of the storage allocation) + if (CubDebug(error = AliasTemporaries(d_temp_storage, temp_storage_bytes, allocations, allocation_sizes))) break; + + // Return if the caller is simply requesting the size of the storage allocation + if (d_temp_storage == NULL) + return cudaSuccess; + + // Privatized per-block reductions + T *d_block_reductions = (T*) allocations[0]; + + // Grid queue descriptor + GridQueue queue(allocations[1]); + + // Prepare the dynamic queue descriptor if necessary + if (privatized_dispatch_params.grid_mapping == GRID_MAPPING_DYNAMIC) + { + // Prepare queue using a kernel so we know it gets prepared once per operation + if (stream_synchronous) CubLog("Invoking prepare_drain_kernel<<<1, 1, 0, %lld>>>()\n", (long long) stream); + + // Invoke prepare_drain_kernel + prepare_drain_kernel<<<1, 1, 0, stream>>>(queue, num_items); + + // Sync the stream if specified + if (stream_synchronous && (CubDebug(error = SyncStream(stream)))) break; + } + + // Log privatized_kernel configuration + if (stream_synchronous) CubLog("Invoking privatized_kernel<<<%d, %d, 0, %lld>>>(), %d items per thread, %d SM occupancy\n", + privatized_grid_size, privatized_dispatch_params.block_threads, (long long) stream, privatized_dispatch_params.items_per_thread, privatized_sm_occupancy); + + // Invoke privatized_kernel + privatized_kernel<<>>( + d_in, + d_block_reductions, + num_items, + even_share, + queue, + reduction_op); + + // Sync the stream if specified + if (stream_synchronous && (CubDebug(error = SyncStream(stream)))) break; + + // Log single_kernel configuration + if (stream_synchronous) CubLog("Invoking single_kernel<<<%d, %d, 0, %lld>>>(), %d items per thread\n", + 1, single_dispatch_params.block_threads, (long long) stream, single_dispatch_params.items_per_thread); + + // Invoke single_kernel + single_kernel<<<1, single_dispatch_params.block_threads, 0, stream>>>( + d_block_reductions, + d_out, + privatized_grid_size, + reduction_op); + + // Sync the stream if specified + if (stream_synchronous && (CubDebug(error = SyncStream(stream)))) break; + } + } + while (0); + + return error; + +#endif // CUB_RUNTIME_ENABLED + } + + + +#endif // DOXYGEN_SHOULD_SKIP_THIS + + /****************************************************************************** + * Interface + ******************************************************************************/ + + /** + * \brief Computes a device-wide reduction using the specified binary \p reduction_op functor. + * + * \par + * Does not support non-commutative reduction operators. + * + * \devicestorage + * + * \cdp + * + * \iterator + * + * \par + * The code snippet below illustrates the max reduction of a device vector of \p int items. + * \par + * \code + * #include + * ... + * + * // Declare and initialize device pointers for input and output + * int *d_reduce_input, *d_aggregate; + * int num_items = ... + * ... + * + * // Determine temporary device storage requirements for reduction + * void *d_temp_storage = NULL; + * size_t temp_storage_bytes = 0; + * cub::DeviceReduce::Reduce(d_temp_storage, temp_storage_bytes, d_reduce_input, d_aggregate, num_items, cub::Max()); + * + * // Allocate temporary storage for reduction + * cudaMalloc(&d_temp_storage, temp_storage_bytes); + * + * // Run reduction (max) + * cub::DeviceReduce::Reduce(d_temp_storage, temp_storage_bytes, d_reduce_input, d_aggregate, num_items, cub::Max()); + * + * \endcode + * + * \tparam InputIteratorRA [inferred] Random-access iterator type for input (may be a simple pointer type) + * \tparam OutputIteratorRA [inferred] Random-access iterator type for output (may be a simple pointer type) + * \tparam ReductionOp [inferred] Binary reduction operator type having member T operator()(const T &a, const T &b) + */ + template < + typename InputIteratorRA, + typename OutputIteratorRA, + typename ReductionOp> + __host__ __device__ __forceinline__ + static cudaError_t Reduce( + void *d_temp_storage, ///< [in] %Device allocation of temporary storage. When NULL, the required allocation size is returned in \p temp_storage_bytes and no work is done. + size_t &temp_storage_bytes, ///< [in,out] Size in bytes of \p d_temp_storage allocation. + InputIteratorRA d_in, ///< [in] Input data to reduce + OutputIteratorRA d_out, ///< [out] Output location for result + int num_items, ///< [in] Number of items to reduce + ReductionOp reduction_op, ///< [in] Binary reduction operator + cudaStream_t stream = 0, ///< [in] [optional] CUDA stream to launch kernels within. Default is stream0. + bool stream_synchronous = false) ///< [in] [optional] Whether or not to synchronize the stream after every kernel launch to check for errors. Default is \p false. + { + // Type used for array indexing + typedef int SizeT; + + // Data type of input iterator + typedef typename std::iterator_traits::value_type T; + + // Tuning polices + typedef PtxDefaultPolicies PtxDefaultPolicies; // Wrapper of default kernel policies + typedef typename PtxDefaultPolicies::PrivatizedPolicy PrivatizedPolicy; // Multi-block kernel policy + typedef typename PtxDefaultPolicies::SinglePolicy SinglePolicy; // Single-block kernel policy + + cudaError error = cudaSuccess; + do + { + // Declare dispatch parameters + KernelDispachParams privatized_dispatch_params; + KernelDispachParams single_dispatch_params; + +#ifdef __CUDA_ARCH__ + // We're on the device, so initialize the dispatch parameters with the PtxDefaultPolicies directly + privatized_dispatch_params.Init(PtxDefaultPolicies::SUBSCRIPTION_FACTOR); + single_dispatch_params.Init(); +#else + // We're on the host, so lookup and initialize the dispatch parameters with the policies that match the device's PTX version + int ptx_version; + if (CubDebug(error = PtxVersion(ptx_version))) break; + PtxDefaultPolicies::InitDispatchParams(ptx_version, privatized_dispatch_params, single_dispatch_params); +#endif + + // Dispatch + if (CubDebug(error = Dispatch( + d_temp_storage, + temp_storage_bytes, + ReducePrivatizedKernel, + ReduceSingleKernel, + ResetDrainKernel, + privatized_dispatch_params, + single_dispatch_params, + d_in, + d_out, + num_items, + reduction_op, + stream, + stream_synchronous))) break; + } + while (0); + + return error; + } + + + /** + * \brief Computes a device-wide sum using the addition ('+') operator. + * + * \par + * Does not support non-commutative reduction operators. + * + * \devicestorage + * + * \cdp + * + * \iterator + * + * \par + * The code snippet below illustrates the sum reduction of a device vector of \p int items. + * \par + * \code + * #include + * ... + * + * // Declare and initialize device pointers for input and output + * int *d_reduce_input, *d_aggregate; + * int num_items = ... + * ... + * + * // Determine temporary device storage requirements for summation + * void *d_temp_storage = NULL; + * size_t temp_storage_bytes = 0; + * cub::DeviceReduce::Sum(d_temp_storage, temp_storage_bytes, d_reduce_input, d_aggregate, num_items); + * + * // Allocate temporary storage for summation + * cudaMalloc(&d_temp_storage, temp_storage_bytes); + * + * // Run reduction summation + * cub::DeviceReduce::Sum(d_temp_storage, temp_storage_bytes, d_reduce_input, d_aggregate, num_items); + * + * \endcode + * + * \tparam InputIteratorRA [inferred] Random-access iterator type for input (may be a simple pointer type) + * \tparam OutputIteratorRA [inferred] Random-access iterator type for output (may be a simple pointer type) + */ + template < + typename InputIteratorRA, + typename OutputIteratorRA> + __host__ __device__ __forceinline__ + static cudaError_t Sum( + void *d_temp_storage, ///< [in] %Device allocation of temporary storage. When NULL, the required allocation size is returned in \p temp_storage_bytes and no work is done. + size_t &temp_storage_bytes, ///< [in,out] Size in bytes of \p d_temp_storage allocation. + InputIteratorRA d_in, ///< [in] Input data to reduce + OutputIteratorRA d_out, ///< [out] Output location for result + int num_items, ///< [in] Number of items to reduce + cudaStream_t stream = 0, ///< [in] [optional] CUDA stream to launch kernels within. Default is stream0. + bool stream_synchronous = false) ///< [in] [optional] Whether or not to synchronize the stream after every kernel launch to check for errors. Default is \p false. + { + return Reduce(d_temp_storage, temp_storage_bytes, d_in, d_out, num_items, cub::Sum(), stream, stream_synchronous); + } + + +}; + + +} // CUB namespace +CUB_NS_POSTFIX // Optional outer namespace(s) + + diff --git a/lib/kokkos/TPL/cub/device/device_reduce_by_key.cuh b/lib/kokkos/TPL/cub/device/device_reduce_by_key.cuh new file mode 100755 index 0000000000..f05f751545 --- /dev/null +++ b/lib/kokkos/TPL/cub/device/device_reduce_by_key.cuh @@ -0,0 +1,633 @@ + +/****************************************************************************** + * Copyright (c) 2011, Duane Merrill. All rights reserved. + * Copyright (c) 2011-2013, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the NVIDIA CORPORATION nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + ******************************************************************************/ + +/** + * \file + * cub::DeviceReduceByKey provides operations for computing a device-wide, parallel prefix scan across data items residing within global memory. + */ + +#pragma once + +#include +#include + +#include "block/block_reduce_by_key_tiles.cuh" +#include "device_scan.cuh" +#include "../thread/thread_operators.cuh" +#include "../grid/grid_queue.cuh" +#include "../util_iterator.cuh" +#include "../util_debug.cuh" +#include "../util_device.cuh" +#include "../util_namespace.cuh" + +/// Optional outer namespace(s) +CUB_NS_PREFIX + +/// CUB namespace +namespace cub { + + +/****************************************************************************** + * Kernel entry points + *****************************************************************************/ + +#ifndef DOXYGEN_SHOULD_SKIP_THIS // Do not document + + +/** + * Reduce-by-key kernel entry point (multi-block) + */ +template < + typename BlockReduceByKeyilesPolicy, ///< Tuning policy for cub::BlockReduceByKeyiles abstraction + typename InputIteratorRA, ///< Random-access iterator type for input (may be a simple pointer type) + typename OutputIteratorRA, ///< Random-access iterator type for output (may be a simple pointer type) + typename T, ///< The scan data type + typename ReductionOp, ///< Binary scan operator type having member T operator()(const T &a, const T &b) + typename Identity, ///< Identity value type (cub::NullType for inclusive scans) + typename SizeT> ///< Integer type used for global array indexing +__launch_bounds__ (int(BlockSweepScanPolicy::BLOCK_THREADS)) +__global__ void MultiBlockScanKernel( + InputIteratorRA d_in, ///< Input data + OutputIteratorRA d_out, ///< Output data + ScanTileDescriptor *d_tile_status, ///< Global list of tile status + ReductionOp reduction_op, ///< Binary scan operator + Identity identity, ///< Identity element + SizeT num_items, ///< Total number of scan items for the entire problem + GridQueue queue) ///< Descriptor for performing dynamic mapping of tile data to thread blocks +{ + enum + { + TILE_STATUS_PADDING = PtxArchProps::WARP_THREADS, + }; + + // Thread block type for scanning input tiles + typedef BlockSweepScan< + BlockSweepScanPolicy, + InputIteratorRA, + OutputIteratorRA, + ReductionOp, + Identity, + SizeT> BlockSweepScanT; + + // Shared memory for BlockSweepScan + __shared__ typename BlockSweepScanT::TempStorage temp_storage; + + // Process tiles + BlockSweepScanT(temp_storage, d_in, d_out, reduction_op, identity).ConsumeTiles( + num_items, + queue, + d_tile_status + TILE_STATUS_PADDING); +} + + +#endif // DOXYGEN_SHOULD_SKIP_THIS + + + +/****************************************************************************** + * DeviceReduceByKey + *****************************************************************************/ + +/** + * \addtogroup DeviceModule + * @{ + */ + +/** + * \brief DeviceReduceByKey provides operations for computing a device-wide, parallel prefix scan across data items residing within global memory. ![](scan_logo.png) + */ +struct DeviceReduceByKey +{ +#ifndef DOXYGEN_SHOULD_SKIP_THIS // Do not document + + /****************************************************************************** + * Constants and typedefs + ******************************************************************************/ + + /// Generic structure for encapsulating dispatch properties. Mirrors the constants within BlockSweepScanPolicy. + struct KernelDispachParams + { + // Policy fields + int block_threads; + int items_per_thread; + BlockLoadAlgorithm load_policy; + BlockStoreAlgorithm store_policy; + BlockScanAlgorithm scan_algorithm; + + // Other misc + int tile_size; + + template + __host__ __device__ __forceinline__ + void Init() + { + block_threads = BlockSweepScanPolicy::BLOCK_THREADS; + items_per_thread = BlockSweepScanPolicy::ITEMS_PER_THREAD; + load_policy = BlockSweepScanPolicy::LOAD_ALGORITHM; + store_policy = BlockSweepScanPolicy::STORE_ALGORITHM; + scan_algorithm = BlockSweepScanPolicy::SCAN_ALGORITHM; + + tile_size = block_threads * items_per_thread; + } + + __host__ __device__ __forceinline__ + void Print() + { + printf("%d, %d, %d, %d, %d", + block_threads, + items_per_thread, + load_policy, + store_policy, + scan_algorithm); + } + + }; + + + /****************************************************************************** + * Tuning policies + ******************************************************************************/ + + + /// Specializations of tuned policy types for different PTX architectures + template < + typename T, + typename SizeT, + int ARCH> + struct TunedPolicies; + + /// SM35 tune + template + struct TunedPolicies + { + typedef BlockSweepScanPolicy<128, 16, BLOCK_LOAD_DIRECT, false, LOAD_LDG, BLOCK_STORE_WARP_TRANSPOSE, true, BLOCK_SCAN_RAKING_MEMOIZE> MultiBlockPolicy; + }; + + /// SM30 tune + template + struct TunedPolicies + { + typedef BlockSweepScanPolicy<256, 9, BLOCK_LOAD_WARP_TRANSPOSE, false, LOAD_DEFAULT, BLOCK_STORE_WARP_TRANSPOSE, false, BLOCK_SCAN_RAKING_MEMOIZE> MultiBlockPolicy; + }; + + /// SM20 tune + template + struct TunedPolicies + { + typedef BlockSweepScanPolicy<128, 15, BLOCK_LOAD_WARP_TRANSPOSE, false, LOAD_DEFAULT, BLOCK_STORE_WARP_TRANSPOSE, false, BLOCK_SCAN_RAKING_MEMOIZE> MultiBlockPolicy; + }; + + /// SM10 tune + template + struct TunedPolicies + { + typedef BlockSweepScanPolicy<128, 7, BLOCK_LOAD_TRANSPOSE, false, LOAD_DEFAULT, BLOCK_STORE_TRANSPOSE, false, BLOCK_SCAN_RAKING> MultiBlockPolicy; + }; + + + /// Tuning policy for the PTX architecture that DeviceReduceByKey operations will get dispatched to + template + struct PtxDefaultPolicies + { + static const int PTX_TUNE_ARCH = (CUB_PTX_ARCH >= 350) ? + 350 : + (CUB_PTX_ARCH >= 300) ? + 300 : + (CUB_PTX_ARCH >= 200) ? + 200 : + 100; + + // Tuned policy set for the current PTX compiler pass + typedef TunedPolicies PtxTunedPolicies; + + // MultiBlockPolicy that opaquely derives from the specialization corresponding to the current PTX compiler pass + struct MultiBlockPolicy : PtxTunedPolicies::MultiBlockPolicy {}; + + /** + * Initialize dispatch params with the policies corresponding to the PTX assembly we will use + */ + static void InitDispatchParams(int ptx_version, KernelDispachParams &multi_block_dispatch_params) + { + if (ptx_version >= 350) + { + typedef TunedPolicies TunedPolicies; + multi_block_dispatch_params.Init(); + } + else if (ptx_version >= 300) + { + typedef TunedPolicies TunedPolicies; + multi_block_dispatch_params.Init(); + } + else if (ptx_version >= 200) + { + typedef TunedPolicies TunedPolicies; + multi_block_dispatch_params.Init(); + } + else + { + typedef TunedPolicies TunedPolicies; + multi_block_dispatch_params.Init(); + } + } + }; + + + /****************************************************************************** + * Utility methods + ******************************************************************************/ + + /** + * Internal dispatch routine + */ + template < + typename InitScanKernelPtr, ///< Function type of cub::InitScanKernel + typename MultiBlockScanKernelPtr, ///< Function type of cub::MultiBlockScanKernel + typename InputIteratorRA, ///< Random-access iterator type for input (may be a simple pointer type) + typename OutputIteratorRA, ///< Random-access iterator type for output (may be a simple pointer type) + typename ReductionOp, ///< Binary scan operator type having member T operator()(const T &a, const T &b) + typename Identity, ///< Identity value type (cub::NullType for inclusive scans) + typename SizeT> ///< Integer type used for global array indexing + __host__ __device__ __forceinline__ + static cudaError_t Dispatch( + void *d_temp_storage, ///< [in] %Device allocation of temporary storage. When NULL, the required allocation size is returned in \p temp_storage_bytes and no work is done. + size_t &temp_storage_bytes, ///< [in,out] Size in bytes of \p d_temp_storage allocation. + InitScanKernelPtr init_kernel, ///< [in] Kernel function pointer to parameterization of cub::InitScanKernel + MultiBlockScanKernelPtr multi_block_kernel, ///< [in] Kernel function pointer to parameterization of cub::MultiBlockScanKernel + KernelDispachParams &multi_block_dispatch_params, ///< [in] Dispatch parameters that match the policy that \p multi_block_kernel was compiled for + InputIteratorRA d_in, ///< [in] Iterator pointing to scan input + OutputIteratorRA d_out, ///< [in] Iterator pointing to scan output + ReductionOp reduction_op, ///< [in] Binary scan operator + Identity identity, ///< [in] Identity element + SizeT num_items, ///< [in] Total number of items to scan + cudaStream_t stream = 0, ///< [in] [optional] CUDA stream to launch kernels within. Default is stream0. + bool stream_synchronous = false) ///< [in] [optional] Whether or not to synchronize the stream after every kernel launch to check for errors. Default is \p false. + { + +#ifndef CUB_RUNTIME_ENABLED + + // Kernel launch not supported from this device + return CubDebug(cudaErrorNotSupported ); + +#else + + enum + { + TILE_STATUS_PADDING = 32, + }; + + // Data type + typedef typename std::iterator_traits::value_type T; + + cudaError error = cudaSuccess; + do + { + // Number of input tiles + int num_tiles = (num_items + multi_block_dispatch_params.tile_size - 1) / multi_block_dispatch_params.tile_size; + + // Temporary storage allocation requirements + void* allocations[2]; + size_t allocation_sizes[2] = + { + (num_tiles + TILE_STATUS_PADDING) * sizeof(ScanTileDescriptor), // bytes needed for tile status descriptors + GridQueue::AllocationSize() // bytes needed for grid queue descriptor + }; + + // Alias temporaries (or set the necessary size of the storage allocation) + if (CubDebug(error = AliasTemporaries(d_temp_storage, temp_storage_bytes, allocations, allocation_sizes))) break; + + // Return if the caller is simply requesting the size of the storage allocation + if (d_temp_storage == NULL) + return cudaSuccess; + + // Global list of tile status + ScanTileDescriptor *d_tile_status = (ScanTileDescriptor*) allocations[0]; + + // Grid queue descriptor + GridQueue queue(allocations[1]); + + // Get GPU id + int device_ordinal; + if (CubDebug(error = cudaGetDevice(&device_ordinal))) break; + + // Get SM count + int sm_count; + if (CubDebug(error = cudaDeviceGetAttribute (&sm_count, cudaDevAttrMultiProcessorCount, device_ordinal))) break; + + // Log init_kernel configuration + int init_kernel_threads = 128; + int init_grid_size = (num_tiles + init_kernel_threads - 1) / init_kernel_threads; + if (stream_synchronous) CubLog("Invoking init_kernel<<<%d, %d, 0, %lld>>>()\n", init_grid_size, init_kernel_threads, (long long) stream); + + // Invoke init_kernel to initialize tile descriptors and queue descriptors + init_kernel<<>>( + queue, + d_tile_status, + num_tiles); + + // Sync the stream if specified +#ifndef __CUDA_ARCH__ + if (stream_synchronous && CubDebug(error = cudaStreamSynchronize(stream))) break; +#else + if (stream_synchronous && CubDebug(error = cudaDeviceSynchronize())) break; +#endif + + // Get a rough estimate of multi_block_kernel SM occupancy based upon the maximum SM occupancy of the targeted PTX architecture + int multi_sm_occupancy = CUB_MIN( + ArchProps::MAX_SM_THREADBLOCKS, + ArchProps::MAX_SM_THREADS / multi_block_dispatch_params.block_threads); + +#ifndef __CUDA_ARCH__ + + // We're on the host, so come up with a more accurate estimate of multi_block_kernel SM occupancy from actual device properties + Device device_props; + if (CubDebug(error = device_props.Init(device_ordinal))) break; + + if (CubDebug(error = device_props.MaxSmOccupancy( + multi_sm_occupancy, + multi_block_kernel, + multi_block_dispatch_params.block_threads))) break; + +#endif + // Get device occupancy for multi_block_kernel + int multi_block_occupancy = multi_sm_occupancy * sm_count; + + // Get grid size for multi_block_kernel + int multi_block_grid_size = (num_tiles < multi_block_occupancy) ? + num_tiles : // Not enough to fill the device with threadblocks + multi_block_occupancy; // Fill the device with threadblocks + + // Log multi_block_kernel configuration + if (stream_synchronous) CubLog("Invoking multi_block_kernel<<<%d, %d, 0, %lld>>>(), %d items per thread, %d SM occupancy\n", + multi_block_grid_size, multi_block_dispatch_params.block_threads, (long long) stream, multi_block_dispatch_params.items_per_thread, multi_sm_occupancy); + + // Invoke multi_block_kernel + multi_block_kernel<<>>( + d_in, + d_out, + d_tile_status, + reduction_op, + identity, + num_items, + queue); + + // Sync the stream if specified +#ifndef __CUDA_ARCH__ + if (stream_synchronous && CubDebug(error = cudaStreamSynchronize(stream))) break; +#else + if (stream_synchronous && CubDebug(error = cudaDeviceSynchronize())) break; +#endif + } + while (0); + + return error; + +#endif // CUB_RUNTIME_ENABLED + } + + + + /** + * Internal scan dispatch routine for using default tuning policies + */ + template < + typename InputIteratorRA, ///< Random-access iterator type for input (may be a simple pointer type) + typename OutputIteratorRA, ///< Random-access iterator type for output (may be a simple pointer type) + typename ReductionOp, ///< Binary scan operator type having member T operator()(const T &a, const T &b) + typename Identity, ///< Identity value type (cub::NullType for inclusive scans) + typename SizeT> ///< Integer type used for global array indexing + __host__ __device__ __forceinline__ + static cudaError_t Dispatch( + void *d_temp_storage, ///< [in] %Device allocation of temporary storage. When NULL, the required allocation size is returned in \p temp_storage_bytes and no work is done. + size_t &temp_storage_bytes, ///< [in,out] Size in bytes of \p d_temp_storage allocation. + InputIteratorRA d_in, ///< [in] Iterator pointing to scan input + OutputIteratorRA d_out, ///< [in] Iterator pointing to scan output + ReductionOp reduction_op, ///< [in] Binary scan operator + Identity identity, ///< [in] Identity element + SizeT num_items, ///< [in] Total number of items to scan + cudaStream_t stream = 0, ///< [in] [optional] CUDA stream to launch kernels within. Default is stream0. + bool stream_synchronous = false) ///< [in] [optional] Whether or not to synchronize the stream after every kernel launch to check for errors. Default is \p false. + { + // Data type + typedef typename std::iterator_traits::value_type T; + + // Tuning polices for the PTX architecture that will get dispatched to + typedef PtxDefaultPolicies PtxDefaultPolicies; + typedef typename PtxDefaultPolicies::MultiBlockPolicy MultiBlockPolicy; + + cudaError error = cudaSuccess; + do + { + // Declare dispatch parameters + KernelDispachParams multi_block_dispatch_params; + +#ifdef __CUDA_ARCH__ + // We're on the device, so initialize the dispatch parameters with the PtxDefaultPolicies directly + multi_block_dispatch_params.Init(); +#else + // We're on the host, so lookup and initialize the dispatch parameters with the policies that match the device's PTX version + int ptx_version; + if (CubDebug(error = PtxVersion(ptx_version))) break; + PtxDefaultPolicies::InitDispatchParams(ptx_version, multi_block_dispatch_params); +#endif + + Dispatch( + d_temp_storage, + temp_storage_bytes, + InitScanKernel, + MultiBlockScanKernel, + multi_block_dispatch_params, + d_in, + d_out, + reduction_op, + identity, + num_items, + stream, + stream_synchronous); + + if (CubDebug(error)) break; + } + while (0); + + return error; + } + + #endif // DOXYGEN_SHOULD_SKIP_THIS + + + /******************************************************************//** + * Interface + *********************************************************************/ + + + /** + * \brief Computes device-wide reductions of consecutive values whose corresponding keys are equal. + * + * The resulting output lists of value-aggregates and their corresponding keys are compacted. + * + * \devicestorage + * + * \tparam KeyInputIteratorRA [inferred] Random-access input iterator type for keys input (may be a simple pointer type) + * \tparam KeyOutputIteratorRA [inferred] Random-access output iterator type for keys output (may be a simple pointer type) + * \tparam ValueInputIteratorRA [inferred] Random-access input iterator type for values input (may be a simple pointer type) + * \tparam ValueOutputIteratorRA [inferred] Random-access output iterator type for values output (may be a simple pointer type) + * \tparam ReductionOp [inferred] Binary reduction operator type having member T operator()(const T &a, const T &b), where \p T is the value type of \p ValueInputIteratorRA + */ + template < + typename KeyInputIteratorRA, + typename KeyOutputIteratorRA, + typename ValueInputIteratorRA, + typename ValueOutputIteratorRA, + typename ReductionOp> + __host__ __device__ __forceinline__ + static cudaError_t ReduceValues( + void *d_temp_storage, ///< [in] %Device allocation of temporary storage. When NULL, the required allocation size is returned in \p temp_storage_bytes and no work is done. + size_t &temp_storage_bytes, ///< [in,out] Size in bytes of \p d_temp_storage allocation. + KeyInputIteratorRA d_keys_in, ///< [in] Key input data + KeyOutputIteratorRA d_keys_out, ///< [out] Key output data (compacted) + ValueInputIteratorRA d_values_in, ///< [in] Value input data + ValueOutputIteratorRA d_values_out, ///< [out] Value output data (compacted) + int num_items, ///< [in] Total number of input pairs + ReductionOp reduction_op, ///< [in] Binary value reduction operator + cudaStream_t stream = 0, ///< [in] [optional] CUDA stream to launch kernels within. Default is stream0. + bool stream_synchronous = false) ///< [in] [optional] Whether or not to synchronize the stream after every kernel launch to check for errors. May cause significant slowdown. Default is \p false. + { + return Dispatch(d_temp_storage, temp_storage_bytes, d_keys_in, d_keys_out, d_values_in, d_values_out, reduction_op, num_items, stream, stream_synchronous); + } + + + /** + * \brief Computes device-wide sums of consecutive values whose corresponding keys are equal. + * + * The resulting output lists of value-aggregates and their corresponding keys are compacted. + * + * \devicestorage + * + * \tparam KeyInputIteratorRA [inferred] Random-access input iterator type for keys input (may be a simple pointer type) + * \tparam KeyOutputIteratorRA [inferred] Random-access output iterator type for keys output (may be a simple pointer type) + * \tparam ValueInputIteratorRA [inferred] Random-access input iterator type for values input (may be a simple pointer type) + * \tparam ValueOutputIteratorRA [inferred] Random-access output iterator type for values output (may be a simple pointer type) + * \tparam ReductionOp [inferred] Binary reduction operator type having member T operator()(const T &a, const T &b), where \p T is the value type of \p ValueInputIteratorRA + */ + template < + typename KeyInputIteratorRA, + typename KeyOutputIteratorRA, + typename ValueInputIteratorRA, + typename ValueOutputIteratorRA> + __host__ __device__ __forceinline__ + static cudaError_t SumValues( + void *d_temp_storage, ///< [in] %Device allocation of temporary storage. When NULL, the required allocation size is returned in \p temp_storage_bytes and no work is done. + size_t &temp_storage_bytes, ///< [in,out] Size in bytes of \p d_temp_storage allocation. + KeyInputIteratorRA d_keys_in, ///< [in] Key input data + KeyOutputIteratorRA d_keys_out, ///< [in] Key output data (compacted) + ValueInputIteratorRA d_values_in, ///< [in] Value input data + ValueOutputIteratorRA d_values_out, ///< [in] Value output data (compacted) + int num_items, ///< [in] Total number of input pairs + cudaStream_t stream = 0, ///< [in] [optional] CUDA stream to launch kernels within. Default is stream0. + bool stream_synchronous = false) ///< [in] [optional] Whether or not to synchronize the stream after every kernel launch to check for errors. May cause significant slowdown. Default is \p false. + { + return ReduceValues(d_temp_storage, temp_storage_bytes, d_keys_in, d_keys_out, d_values_in, d_values_out, cub::Sum(), num_items, stream, stream_synchronous); + } + + + /** + * \brief Computes the "run-length" of each group of consecutive, equal-valued keys. + * + * The resulting output lists of run-length counts and their corresponding keys are compacted. + * + * \devicestorage + * + * \tparam KeyInputIteratorRA [inferred] Random-access input iterator type for keys input (may be a simple pointer type) + * \tparam KeyOutputIteratorRA [inferred] Random-access output iterator type for keys output (may be a simple pointer type) + * \tparam CountOutputIteratorRA [inferred] Random-access output iterator type for output of key-counts whose value type must be convertible to an integer type (may be a simple pointer type) + */ + template < + typename KeyInputIteratorRA, + typename KeyOutputIteratorRA, + typename CountOutputIteratorRA> + __host__ __device__ __forceinline__ + static cudaError_t RunLengths( + void *d_temp_storage, ///< [in] %Device allocation of temporary storage. When NULL, the required allocation size is returned in \p temp_storage_bytes and no work is done. + size_t &temp_storage_bytes, ///< [in,out] Size in bytes of \p d_temp_storage allocation. + KeyInputIteratorRA d_keys_in, ///< [in] Key input data + KeyOutputIteratorRA d_keys_out, ///< [in] Key output data (compacted) + CountOutputIteratorRA d_counts_out, ///< [in] Run-length counts output data (compacted) + int num_items, ///< [in] Total number of keys + cudaStream_t stream = 0, ///< [in] [optional] CUDA stream to launch kernels within. Default is stream0. + bool stream_synchronous = false) ///< [in] [optional] Whether or not to synchronize the stream after every kernel launch to check for errors. May cause significant slowdown. Default is \p false. + { + typedef typename std::iterator_traits::value_type CountT; + return SumValues(d_temp_storage, temp_storage_bytes, d_keys_in, d_keys_out, ConstantIteratorRA(1), d_counts_out, num_items, stream, stream_synchronous); + } + + + /** + * \brief Removes duplicates within each group of consecutive, equal-valued keys. Only the first key from each group (and corresponding value) is kept. + * + * The resulting keys are compacted. + * + * \devicestorage + * + * \tparam KeyInputIteratorRA [inferred] Random-access input iterator type for keys input (may be a simple pointer type) + * \tparam KeyOutputIteratorRA [inferred] Random-access output iterator type for keys output (may be a simple pointer type) + * \tparam ValueInputIteratorRA [inferred] Random-access input iterator type for values input (may be a simple pointer type) + * \tparam ValueOutputIteratorRA [inferred] Random-access output iterator type for values output (may be a simple pointer type) + * \tparam ReductionOp [inferred] Binary reduction operator type having member T operator()(const T &a, const T &b), where \p T is the value type of \p ValueInputIteratorRA + */ + template < + typename KeyInputIteratorRA, + typename KeyOutputIteratorRA, + typename ValueInputIteratorRA, + typename ValueOutputIteratorRA, + typename ReductionOp> + __host__ __device__ __forceinline__ + static cudaError_t Unique( + void *d_temp_storage, ///< [in] %Device allocation of temporary storage. When NULL, the required allocation size is returned in \p temp_storage_bytes and no work is done. + size_t &temp_storage_bytes, ///< [in,out] Size in bytes of \p d_temp_storage allocation. + KeyInputIteratorRA d_keys_in, ///< [in] Key input data + KeyOutputIteratorRA d_keys_out, ///< [out] Key output data (compacted) + ValueInputIteratorRA d_values_in, ///< [in] Value input data + ValueOutputIteratorRA d_values_out, ///< [out] Value output data (compacted) + int num_items, ///< [in] Total number of input pairs + cudaStream_t stream = 0, ///< [in] [optional] CUDA stream to launch kernels within. Default is stream0. + bool stream_synchronous = false) ///< [in] [optional] Whether or not to synchronize the stream after every kernel launch to check for errors. May cause significant slowdown. Default is \p false. + { + return Dispatch(d_temp_storage, temp_storage_bytes, d_keys_in, d_keys_out, d_values_in, d_values_out, reduction_op, num_items, stream, stream_synchronous); + } + + + +}; + + +/** @} */ // DeviceModule + +} // CUB namespace +CUB_NS_POSTFIX // Optional outer namespace(s) + + diff --git a/lib/kokkos/TPL/cub/device/device_reorder.cuh b/lib/kokkos/TPL/cub/device/device_reorder.cuh new file mode 100755 index 0000000000..cba3bb48f1 --- /dev/null +++ b/lib/kokkos/TPL/cub/device/device_reorder.cuh @@ -0,0 +1,550 @@ + +/****************************************************************************** + * Copyright (c) 2011, Duane Merrill. All rights reserved. + * Copyright (c) 2011-2013, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the NVIDIA CORPORATION nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + ******************************************************************************/ + +/** + * \file + * cub::DeviceReorder provides device-wide operations for partitioning and filtering lists of items residing within global memory. + */ + +#pragma once + +#include +#include + +#include "device_scan.cuh" +#include "block/block_partition_tiles.cuh" +#include "../grid/grid_queue.cuh" +#include "../util_debug.cuh" +#include "../util_device.cuh" +#include "../util_vector.cuh" +#include "../util_namespace.cuh" + +/// Optional outer namespace(s) +CUB_NS_PREFIX + +/// CUB namespace +namespace cub { + + +/****************************************************************************** + * Kernel entry points + *****************************************************************************/ + +#ifndef DOXYGEN_SHOULD_SKIP_THIS // Do not document + +/** + * Partition kernel entry point (multi-block) + */ +template < + typename BlockPartitionTilesPolicy, ///< Tuning policy for cub::BlockPartitionTiles abstraction + typename InputIteratorRA, ///< Random-access iterator type for input (may be a simple pointer type) + typename OutputIteratorRA, ///< Random-access iterator type for output (may be a simple pointer type) + typename LengthOutputIterator, ///< Output iterator type for recording the length of the first partition (may be a simple pointer type) + typename PredicateOp, ///< Unary predicate operator indicating membership in the first partition type having member bool operator()(const T &val) + typename SizeT> ///< Integer type used for global array indexing +__launch_bounds__ (int(BlockPartitionTilesPolicy::BLOCK_THREADS)) +__global__ void PartitionKernel( + InputIteratorRA d_in, ///< Input data + OutputIteratorRA d_out, ///< Output data + LengthOutputIterator d_partition_length, ///< Number of items in the first partition + ScanTileDescriptor > *d_tile_status, ///< Global list of tile status + PredicateOp pred_op, ///< Unary predicate operator indicating membership in the first partition + SizeT num_items, ///< Total number of input items for the entire problem + int num_tiles, ///< Totla number of intut tiles for the entire problem + GridQueue queue) ///< Descriptor for performing dynamic mapping of tile data to thread blocks +{ + enum + { + TILE_STATUS_PADDING = PtxArchProps::WARP_THREADS, + }; + + typedef PartitionScanTuple PartitionScanTuple; + + // Thread block type for scanning input tiles + typedef BlockPartitionTiles< + BlockPartitionTilesPolicy, + InputIteratorRA, + OutputIteratorRA, + PredicateOp, + SizeT> BlockPartitionTilesT; + + // Shared memory for BlockPartitionTiles + __shared__ typename BlockPartitionTilesT::TempStorage temp_storage; + + // Process tiles + PartitionScanTuple partition_ends; // Ending offsets for partitions (one-after) + bool is_last_tile; // Whether or not this block handled the last tile (i.e., partition_ends is valid for the entire input) + BlockPartitionTilesT(temp_storage, d_in, d_out, d_tile_status + TILE_STATUS_PADDING, pred_op, num_items).ConsumeTiles( + queue, + num_tiles, + partition_ends, + is_last_tile); + + // Record the length of the first partition + if (is_last_tile && (threadIdx.x == 0)) + { + *d_partition_length = partition_ends.x; + } +} + + +#endif // DOXYGEN_SHOULD_SKIP_THIS + + + +/****************************************************************************** + * DeviceReorder + *****************************************************************************/ + +/** + * \addtogroup DeviceModule + * @{ + */ + +/** + * \brief DeviceReorder provides device-wide operations for partitioning and filtering lists of items residing within global memory + */ +struct DeviceReorder +{ +#ifndef DOXYGEN_SHOULD_SKIP_THIS // Do not document + + /****************************************************************************** + * Constants and typedefs + ******************************************************************************/ + + /// Generic structure for encapsulating dispatch properties. Mirrors the constants within BlockPartitionTilesPolicy. + struct KernelDispachParams + { + int block_threads; + int items_per_thread; + BlockScanAlgorithm scan_algorithm; + int tile_size; + + template + __host__ __device__ __forceinline__ + void Init() + { + block_threads = BlockPartitionTilesPolicy::BLOCK_THREADS; + items_per_thread = BlockPartitionTilesPolicy::ITEMS_PER_THREAD; + scan_algorithm = BlockPartitionTilesPolicy::SCAN_ALGORITHM; + tile_size = block_threads * items_per_thread; + } + }; + + + /****************************************************************************** + * Tuning policies + ******************************************************************************/ + + + /// Specializations of tuned policy types for different PTX architectures + template < + int PARTITIONS, + typename T, + typename SizeT, + int ARCH> + struct TunedPolicies; + + /// SM35 tune + template + struct TunedPolicies + { + enum { + NOMINAL_4B_ITEMS_PER_THREAD = 16, + ITEMS_PER_THREAD = CUB_MIN(NOMINAL_4B_ITEMS_PER_THREAD, CUB_MAX(1, (NOMINAL_4B_ITEMS_PER_THREAD * 4 / sizeof(T)))), + }; + + typedef BlockPartitionTilesPolicy PartitionPolicy; + }; + + /// SM30 tune + template + struct TunedPolicies + { + enum { + NOMINAL_4B_ITEMS_PER_THREAD = 9, + ITEMS_PER_THREAD = CUB_MIN(NOMINAL_4B_ITEMS_PER_THREAD, CUB_MAX(1, (NOMINAL_4B_ITEMS_PER_THREAD * 4 / sizeof(T)))), + }; + + typedef BlockPartitionTilesPolicy PartitionPolicy; + }; + + /// SM20 tune + template + struct TunedPolicies + { + enum { + NOMINAL_4B_ITEMS_PER_THREAD = 15, + ITEMS_PER_THREAD = CUB_MIN(NOMINAL_4B_ITEMS_PER_THREAD, CUB_MAX(1, (NOMINAL_4B_ITEMS_PER_THREAD * 4 / sizeof(T)))), + }; + + typedef BlockPartitionTilesPolicy PartitionPolicy; + }; + + /// SM10 tune + template + struct TunedPolicies + { + enum { + NOMINAL_4B_ITEMS_PER_THREAD = 7, + ITEMS_PER_THREAD = CUB_MIN(NOMINAL_4B_ITEMS_PER_THREAD, CUB_MAX(1, (NOMINAL_4B_ITEMS_PER_THREAD * 4 / sizeof(T)))), + }; + typedef BlockPartitionTilesPolicy PartitionPolicy; + }; + + + /// Tuning policy for the PTX architecture that DevicePartition operations will get dispatched to + template + struct PtxDefaultPolicies + { + static const int PTX_TUNE_ARCH = (CUB_PTX_ARCH >= 350) ? + 350 : + (CUB_PTX_ARCH >= 300) ? + 300 : + (CUB_PTX_ARCH >= 200) ? + 200 : + 100; + + // Tuned policy set for the current PTX compiler pass + typedef TunedPolicies PtxTunedPolicies; + + // PartitionPolicy that opaquely derives from the specialization corresponding to the current PTX compiler pass + struct PartitionPolicy : PtxTunedPolicies::PartitionPolicy {}; + + /** + * Initialize dispatch params with the policies corresponding to the PTX assembly we will use + */ + static void InitDispatchParams(int ptx_version, KernelDispachParams &scan_dispatch_params) + { + if (ptx_version >= 350) + { + typedef TunedPolicies TunedPolicies; + scan_dispatch_params.Init(); + } + else if (ptx_version >= 300) + { + typedef TunedPolicies TunedPolicies; + scan_dispatch_params.Init(); + } + else if (ptx_version >= 200) + { + typedef TunedPolicies TunedPolicies; + scan_dispatch_params.Init(); + } + else + { + typedef TunedPolicies TunedPolicies; + scan_dispatch_params.Init(); + } + } + }; + + + /****************************************************************************** + * Utility methods + ******************************************************************************/ + + /** + * Internal dispatch routine + */ + template < + typename ScanInitKernelPtr, ///< Function type of cub::ScanInitKernel + typename PartitionKernelPtr, ///< Function type of cub::PartitionKernel + typename InputIteratorRA, ///< Random-access iterator type for input (may be a simple pointer type) + typename OutputIteratorRA, ///< Random-access iterator type for output (may be a simple pointer type) + typename LengthOutputIterator, ///< Output iterator type for recording the length of the first partition (may be a simple pointer type) + typename PredicateOp, ///< Unary predicate operator indicating membership in the first partition type having member bool operator()(const T &val) + typename SizeT> ///< Integer type used for global array indexing + __host__ __device__ __forceinline__ + static cudaError_t Dispatch( + int ptx_version, ///< [in] PTX version + void *d_temp_storage, ///< [in] %Device allocation of temporary storage. When NULL, the required allocation size is returned in \p temp_storage_bytes and no work is done. + size_t &temp_storage_bytes, ///< [in,out] Size in bytes of \p d_temp_storage allocation. + ScanInitKernelPtr init_kernel, ///< [in] Kernel function pointer to parameterization of cub::PartitionInitKernel + PartitionKernelPtr partition_kernel, ///< [in] Kernel function pointer to parameterization of cub::PartitionKernel + KernelDispachParams &scan_dispatch_params, ///< [in] Dispatch parameters that match the policy that \p partition_kernel was compiled for + InputIteratorRA d_in, ///< [in] Iterator pointing to scan input + OutputIteratorRA d_out, ///< [in] Iterator pointing to scan output + LengthOutputIterator d_partition_length, ///< [out] Output iterator referencing the location where the pivot offset (i.e., the length of the first partition) is to be recorded + PredicateOp pred_op, ///< [in] Unary predicate operator indicating membership in the first partition + SizeT num_items, ///< [in] Total number of items to partition + cudaStream_t stream = 0, ///< [in] [optional] CUDA stream to launch kernels within. Default is stream0. + bool stream_synchronous = false) ///< [in] [optional] Whether or not to synchronize the stream after every kernel launch to check for errors. Default is \p false. + { + +#ifndef CUB_RUNTIME_ENABLED + + // Kernel launch not supported from this device + return CubDebug(cudaErrorNotSupported); + +#else + + enum + { + TILE_STATUS_PADDING = 32, + }; + + // Data type + typedef typename std::iterator_traits::value_type T; + + // Scan tuple type and tile status descriptor type + typedef typename VectorHelper::Type ScanTuple; + typedef ScanTileDescriptor ScanTileDescriptorT; + + cudaError error = cudaSuccess; + do + { + // Number of input tiles + int num_tiles = (num_items + scan_dispatch_params.tile_size - 1) / scan_dispatch_params.tile_size; + + // Temporary storage allocation requirements + void* allocations[2]; + size_t allocation_sizes[2] = + { + (num_tiles + TILE_STATUS_PADDING) * sizeof(ScanTileDescriptorT), // bytes needed for tile status descriptors + GridQueue::AllocationSize() // bytes needed for grid queue descriptor + }; + + // Alias temporaries (or set the necessary size of the storage allocation) + if (CubDebug(error = AliasTemporaries(d_temp_storage, temp_storage_bytes, allocations, allocation_sizes))) break; + + // Return if the caller is simply requesting the size of the storage allocation + if (d_temp_storage == NULL) + return cudaSuccess; + + // Global list of tile status + ScanTileDescriptorT *d_tile_status = (ScanTileDescriptorT*) allocations[0]; + + // Grid queue descriptor + GridQueue queue(allocations[1]); + + // Log init_kernel configuration + int init_kernel_threads = 128; + int init_grid_size = (num_tiles + init_kernel_threads - 1) / init_kernel_threads; + if (stream_synchronous) CubLog("Invoking init_kernel<<<%d, %d, 0, %lld>>>()\n", init_grid_size, init_kernel_threads, (long long) stream); + + // Invoke init_kernel to initialize tile descriptors and queue descriptors + init_kernel<<>>( + queue, + d_tile_status, + num_tiles); + + // Sync the stream if specified + if (stream_synchronous && (CubDebug(error = SyncStream(stream)))) break; + + // Get grid size for multi-block kernel + int scan_grid_size; + int multi_sm_occupancy = -1; + if (ptx_version < 200) + { + // We don't have atomics (or don't have fast ones), so just assign one + // block per tile (limited to 65K tiles) + scan_grid_size = num_tiles; + } + else + { + // We have atomics and can thus reuse blocks across multiple tiles using a queue descriptor. + // Get GPU id + int device_ordinal; + if (CubDebug(error = cudaGetDevice(&device_ordinal))) break; + + // Get SM count + int sm_count; + if (CubDebug(error = cudaDeviceGetAttribute (&sm_count, cudaDevAttrMultiProcessorCount, device_ordinal))) break; + + // Get a rough estimate of partition_kernel SM occupancy based upon the maximum SM occupancy of the targeted PTX architecture + multi_sm_occupancy = CUB_MIN( + ArchProps::MAX_SM_THREADBLOCKS, + ArchProps::MAX_SM_THREADS / scan_dispatch_params.block_threads); + +#ifndef __CUDA_ARCH__ + // We're on the host, so come up with a + Device device_props; + if (CubDebug(error = device_props.Init(device_ordinal))) break; + + if (CubDebug(error = device_props.MaxSmOccupancy( + multi_sm_occupancy, + partition_kernel, + scan_dispatch_params.block_threads))) break; +#endif + // Get device occupancy for partition_kernel + int scan_occupancy = multi_sm_occupancy * sm_count; + + // Get grid size for partition_kernel + scan_grid_size = (num_tiles < scan_occupancy) ? + num_tiles : // Not enough to fill the device with threadblocks + scan_occupancy; // Fill the device with threadblocks + } + + // Log partition_kernel configuration + if (stream_synchronous) CubLog("Invoking partition_kernel<<<%d, %d, 0, %lld>>>(), %d items per thread, %d SM occupancy\n", + scan_grid_size, scan_dispatch_params.block_threads, (long long) stream, scan_dispatch_params.items_per_thread, multi_sm_occupancy); + + // Invoke partition_kernel + partition_kernel<<>>( + d_in, + d_out, + d_partition_length, + d_tile_status, + pred_op, + num_items, + num_tiles, + queue); + + // Sync the stream if specified + if (stream_synchronous && (CubDebug(error = SyncStream(stream)))) break; + } + while (0); + + return error; + +#endif // CUB_RUNTIME_ENABLED + } + + + + /** + * Internal partition dispatch routine for using default tuning policies + */ + template < + typename PARTITIONS, ///< Number of partitions we are keeping + typename InputIteratorRA, ///< Random-access iterator type for input (may be a simple pointer type) + typename OutputIteratorRA, ///< Random-access iterator type for output (may be a simple pointer type) + typename LengthOutputIterator, ///< Output iterator type for recording the length of the first partition (may be a simple pointer type) + typename PredicateOp, ///< Unary predicate operator indicating membership in the first partition type having member bool operator()(const T &val) + typename SizeT> ///< Integer type used for global array indexing + __host__ __device__ __forceinline__ + static cudaError_t Dispatch( + void *d_temp_storage, ///< [in] %Device allocation of temporary storage. When NULL, the required allocation size is returned in \p temp_storage_bytes and no work is done. + size_t &temp_storage_bytes, ///< [in,out] Size in bytes of \p d_temp_storage allocation. + InputIteratorRA d_in, ///< [in] Iterator pointing to input items + OutputIteratorRA d_out, ///< [in] Iterator pointing to output items + LengthOutputIterator d_partition_length, ///< [out] Output iterator referencing the location where the pivot offset (i.e., the length of the first partition) is to be recorded + PredicateOp pred_op, ///< [in] Unary predicate operator indicating membership in the first partition + SizeT num_items, ///< [in] Total number of items to partition + cudaStream_t stream = 0, ///< [in] [optional] CUDA stream to launch kernels within. Default is stream0. + bool stream_synchronous = false) ///< [in] [optional] Whether or not to synchronize the stream after every kernel launch to check for errors. Default is \p false. + { + // Data type + typedef typename std::iterator_traits::value_type T; + + // Tuning polices + typedef PtxDefaultPolicies PtxDefaultPolicies; // Wrapper of default kernel policies + typedef typename PtxDefaultPolicies::PartitionPolicy PartitionPolicy; // Partition kernel policy + + cudaError error = cudaSuccess; + do + { + // Declare dispatch parameters + KernelDispachParams scan_dispatch_params; + + int ptx_version; +#ifdef __CUDA_ARCH__ + // We're on the device, so initialize the dispatch parameters with the PtxDefaultPolicies directly + scan_dispatch_params.Init(); + ptx_version = CUB_PTX_ARCH; +#else + // We're on the host, so lookup and initialize the dispatch parameters with the policies that match the device's PTX version + if (CubDebug(error = PtxVersion(ptx_version))) break; + PtxDefaultPolicies::InitDispatchParams(ptx_version, scan_dispatch_params); +#endif + + Dispatch( + ptx_version, + d_temp_storage, + temp_storage_bytes, + ScanInitKernel, + PartitionKernel, + scan_dispatch_params, + d_in, + d_out, + d_partition_length, + pred_op, + num_items, + stream, + stream_synchronous); + + if (CubDebug(error)) break; + } + while (0); + + return error; + } + + #endif // DOXYGEN_SHOULD_SKIP_THIS + + + /** + * \brief Splits a list of input items into two partitions within the given output list using the specified predicate. The relative ordering of inputs is not necessarily preserved. + * + * An item \p val is placed in the first partition if pred_op(val) == true, otherwise + * it is placed in the second partition. The offset of the partitioning pivot (equivalent to + * the total length of the first partition as well as the starting offset of the second), is + * recorded to \p d_partition_length. + * + * The length of the output referenced by \p d_out is assumed to be the same as that of \p d_in. + * + * \devicestorage + * + * \tparam InputIteratorRA [inferred] Random-access iterator type for input (may be a simple pointer type) + * \tparam OutputIteratorRA [inferred] Random-access iterator type for output (may be a simple pointer type) + * \tparam LengthOutputIterator [inferred] Random-access iterator type for output (may be a simple pointer type) + * \tparam PredicateOp [inferred] Unary predicate operator indicating membership in the first partition type having member bool operator()(const T &val) + */ + template < + typename InputIteratorRA, + typename OutputIteratorRA, + typename LengthOutputIterator, + typename PredicateOp> + __host__ __device__ __forceinline__ + static cudaError_t Partition( + void *d_temp_storage, ///< [in] %Device allocation of temporary storage. When NULL, the required allocation size is returned in \p temp_storage_bytes and no work is done. + size_t &temp_storage_bytes, ///< [in,out] Size in bytes of \p d_temp_storage allocation. + InputIteratorRA d_in, ///< [in] Iterator pointing to input items + OutputIteratorRA d_out, ///< [in] Iterator pointing to output items + LengthOutputIterator d_pivot_offset, ///< [out] Output iterator referencing the location where the pivot offset is to be recorded + PredicateOp pred_op, ///< [in] Unary predicate operator indicating membership in the first partition + int num_items, ///< [in] Total number of items to partition + cudaStream_t stream = 0, ///< [in] [optional] CUDA stream to launch kernels within. Default is stream0. + bool stream_synchronous = false) ///< [in] [optional] Whether or not to synchronize the stream after every kernel launch to check for errors. May cause significant slowdown. Default is \p false. + { + typedef typename std::iterator_traits::value_type T; + return Dispatch(d_temp_storage, temp_storage_bytes, d_in, d_out, Sum(), T(), num_items, stream, stream_synchronous); + } + + +}; + + +/** @} */ // DeviceModule + +} // CUB namespace +CUB_NS_POSTFIX // Optional outer namespace(s) + + diff --git a/lib/kokkos/TPL/cub/device/device_scan.cuh b/lib/kokkos/TPL/cub/device/device_scan.cuh new file mode 100755 index 0000000000..c0640c857e --- /dev/null +++ b/lib/kokkos/TPL/cub/device/device_scan.cuh @@ -0,0 +1,812 @@ + +/****************************************************************************** + * Copyright (c) 2011, Duane Merrill. All rights reserved. + * Copyright (c) 2011-2013, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the NVIDIA CORPORATION nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + ******************************************************************************/ + +/** + * \file + * cub::DeviceScan provides operations for computing a device-wide, parallel prefix scan across data items residing within global memory. + */ + +#pragma once + +#include +#include + +#include "block/block_scan_tiles.cuh" +#include "../thread/thread_operators.cuh" +#include "../grid/grid_queue.cuh" +#include "../util_debug.cuh" +#include "../util_device.cuh" +#include "../util_namespace.cuh" + +/// Optional outer namespace(s) +CUB_NS_PREFIX + +/// CUB namespace +namespace cub { + + +/****************************************************************************** + * Kernel entry points + *****************************************************************************/ + +#ifndef DOXYGEN_SHOULD_SKIP_THIS // Do not document + + +/** + * Initialization kernel for tile status initialization (multi-block) + */ +template < + typename T, ///< Scan value type + typename SizeT> ///< Integer type used for global array indexing +__global__ void ScanInitKernel( + GridQueue grid_queue, ///< [in] Descriptor for performing dynamic mapping of input tiles to thread blocks + ScanTileDescriptor *d_tile_status, ///< [out] Tile status words + int num_tiles) ///< [in] Number of tiles +{ + typedef ScanTileDescriptor ScanTileDescriptorT; + + enum + { + TILE_STATUS_PADDING = PtxArchProps::WARP_THREADS, + }; + + // Reset queue descriptor + if ((blockIdx.x == 0) && (threadIdx.x == 0)) grid_queue.ResetDrain(num_tiles); + + // Initialize tile status + int tile_offset = (blockIdx.x * blockDim.x) + threadIdx.x; + if (tile_offset < num_tiles) + { + // Not-yet-set + d_tile_status[TILE_STATUS_PADDING + tile_offset].status = SCAN_TILE_INVALID; + } + + if ((blockIdx.x == 0) && (threadIdx.x < TILE_STATUS_PADDING)) + { + // Padding + d_tile_status[threadIdx.x].status = SCAN_TILE_OOB; + } +} + + +/** + * Scan kernel entry point (multi-block) + */ +template < + typename BlockScanTilesPolicy, ///< Tuning policy for cub::BlockScanTiles abstraction + typename InputIteratorRA, ///< Random-access iterator type for input (may be a simple pointer type) + typename OutputIteratorRA, ///< Random-access iterator type for output (may be a simple pointer type) + typename T, ///< The scan data type + typename ScanOp, ///< Binary scan operator type having member T operator()(const T &a, const T &b) + typename Identity, ///< Identity value type (cub::NullType for inclusive scans) + typename SizeT> ///< Integer type used for global array indexing +__launch_bounds__ (int(BlockScanTilesPolicy::BLOCK_THREADS)) +__global__ void ScanKernel( + InputIteratorRA d_in, ///< Input data + OutputIteratorRA d_out, ///< Output data + ScanTileDescriptor *d_tile_status, ///< Global list of tile status + ScanOp scan_op, ///< Binary scan operator + Identity identity, ///< Identity element + SizeT num_items, ///< Total number of scan items for the entire problem + GridQueue queue) ///< Descriptor for performing dynamic mapping of tile data to thread blocks +{ + enum + { + TILE_STATUS_PADDING = PtxArchProps::WARP_THREADS, + }; + + // Thread block type for scanning input tiles + typedef BlockScanTiles< + BlockScanTilesPolicy, + InputIteratorRA, + OutputIteratorRA, + ScanOp, + Identity, + SizeT> BlockScanTilesT; + + // Shared memory for BlockScanTiles + __shared__ typename BlockScanTilesT::TempStorage temp_storage; + + // Process tiles + BlockScanTilesT(temp_storage, d_in, d_out, scan_op, identity).ConsumeTiles( + num_items, + queue, + d_tile_status + TILE_STATUS_PADDING); +} + + +#endif // DOXYGEN_SHOULD_SKIP_THIS + + + +/****************************************************************************** + * DeviceScan + *****************************************************************************/ + +/** + * \brief DeviceScan provides operations for computing a device-wide, parallel prefix scan across data items residing within global memory. ![](device_scan.png) + * \ingroup DeviceModule + * + * \par Overview + * Given a list of input elements and a binary reduction operator, a [prefix scan](http://en.wikipedia.org/wiki/Prefix_sum) + * produces an output list where each element is computed to be the reduction + * of the elements occurring earlier in the input list. Prefix sum + * connotes a prefix scan with the addition operator. The term \em inclusive indicates + * that the ith output reduction incorporates the ith input. + * The term \em exclusive indicates the ith input is not incorporated into + * the ith output reduction. + * + * \par Usage Considerations + * \cdp_class{DeviceScan} + * + * \par Performance + * + * \image html scan_perf.png + * + */ +struct DeviceScan +{ +#ifndef DOXYGEN_SHOULD_SKIP_THIS // Do not document + + /****************************************************************************** + * Constants and typedefs + ******************************************************************************/ + + /// Generic structure for encapsulating dispatch properties. Mirrors the constants within BlockScanTilesPolicy. + struct KernelDispachParams + { + // Policy fields + int block_threads; + int items_per_thread; + BlockLoadAlgorithm load_policy; + BlockStoreAlgorithm store_policy; + BlockScanAlgorithm scan_algorithm; + + // Other misc + int tile_size; + + template + __host__ __device__ __forceinline__ + void Init() + { + block_threads = BlockScanTilesPolicy::BLOCK_THREADS; + items_per_thread = BlockScanTilesPolicy::ITEMS_PER_THREAD; + load_policy = BlockScanTilesPolicy::LOAD_ALGORITHM; + store_policy = BlockScanTilesPolicy::STORE_ALGORITHM; + scan_algorithm = BlockScanTilesPolicy::SCAN_ALGORITHM; + + tile_size = block_threads * items_per_thread; + } + + __host__ __device__ __forceinline__ + void Print() + { + printf("%d, %d, %d, %d, %d", + block_threads, + items_per_thread, + load_policy, + store_policy, + scan_algorithm); + } + + }; + + + /****************************************************************************** + * Tuning policies + ******************************************************************************/ + + + /// Specializations of tuned policy types for different PTX architectures + template < + typename T, + typename SizeT, + int ARCH> + struct TunedPolicies; + + /// SM35 tune + template + struct TunedPolicies + { + enum { + NOMINAL_4B_ITEMS_PER_THREAD = 16, + ITEMS_PER_THREAD = CUB_MIN(NOMINAL_4B_ITEMS_PER_THREAD, CUB_MAX(1, (NOMINAL_4B_ITEMS_PER_THREAD * 4 / sizeof(T)))), + }; + + // ScanPolicy: GTX Titan: 29.1B items/s (232.4 GB/s) @ 48M 32-bit T + typedef BlockScanTilesPolicy<128, ITEMS_PER_THREAD, BLOCK_LOAD_DIRECT, false, LOAD_LDG, BLOCK_STORE_WARP_TRANSPOSE, true, BLOCK_SCAN_RAKING_MEMOIZE> ScanPolicy; + }; + + /// SM30 tune + template + struct TunedPolicies + { + enum { + NOMINAL_4B_ITEMS_PER_THREAD = 9, + ITEMS_PER_THREAD = CUB_MIN(NOMINAL_4B_ITEMS_PER_THREAD, CUB_MAX(1, (NOMINAL_4B_ITEMS_PER_THREAD * 4 / sizeof(T)))), + }; + + typedef BlockScanTilesPolicy<256, ITEMS_PER_THREAD, BLOCK_LOAD_WARP_TRANSPOSE, false, LOAD_DEFAULT, BLOCK_STORE_WARP_TRANSPOSE, false, BLOCK_SCAN_RAKING_MEMOIZE> ScanPolicy; + }; + + /// SM20 tune + template + struct TunedPolicies + { + enum { + NOMINAL_4B_ITEMS_PER_THREAD = 15, + ITEMS_PER_THREAD = CUB_MIN(NOMINAL_4B_ITEMS_PER_THREAD, CUB_MAX(1, (NOMINAL_4B_ITEMS_PER_THREAD * 4 / sizeof(T)))), + }; + + // ScanPolicy: GTX 580: 20.3B items/s (162.3 GB/s) @ 48M 32-bit T + typedef BlockScanTilesPolicy<128, ITEMS_PER_THREAD, BLOCK_LOAD_WARP_TRANSPOSE, false, LOAD_DEFAULT, BLOCK_STORE_WARP_TRANSPOSE, false, BLOCK_SCAN_RAKING_MEMOIZE> ScanPolicy; + }; + + /// SM10 tune + template + struct TunedPolicies + { + enum { + NOMINAL_4B_ITEMS_PER_THREAD = 7, + ITEMS_PER_THREAD = CUB_MIN(NOMINAL_4B_ITEMS_PER_THREAD, CUB_MAX(1, (NOMINAL_4B_ITEMS_PER_THREAD * 4 / sizeof(T)))), + }; + typedef BlockScanTilesPolicy<128, ITEMS_PER_THREAD, BLOCK_LOAD_TRANSPOSE, false, LOAD_DEFAULT, BLOCK_STORE_TRANSPOSE, false, BLOCK_SCAN_RAKING> ScanPolicy; + }; + + + /// Tuning policy for the PTX architecture that DeviceScan operations will get dispatched to + template + struct PtxDefaultPolicies + { + static const int PTX_TUNE_ARCH = (CUB_PTX_ARCH >= 350) ? + 350 : + (CUB_PTX_ARCH >= 300) ? + 300 : + (CUB_PTX_ARCH >= 200) ? + 200 : + 100; + + // Tuned policy set for the current PTX compiler pass + typedef TunedPolicies PtxTunedPolicies; + + // ScanPolicy that opaquely derives from the specialization corresponding to the current PTX compiler pass + struct ScanPolicy : PtxTunedPolicies::ScanPolicy {}; + + /** + * Initialize dispatch params with the policies corresponding to the PTX assembly we will use + */ + static void InitDispatchParams(int ptx_version, KernelDispachParams &scan_dispatch_params) + { + if (ptx_version >= 350) + { + typedef TunedPolicies TunedPolicies; + scan_dispatch_params.Init(); + } + else if (ptx_version >= 300) + { + typedef TunedPolicies TunedPolicies; + scan_dispatch_params.Init(); + } + else if (ptx_version >= 200) + { + typedef TunedPolicies TunedPolicies; + scan_dispatch_params.Init(); + } + else + { + typedef TunedPolicies TunedPolicies; + scan_dispatch_params.Init(); + } + } + }; + + + /****************************************************************************** + * Utility methods + ******************************************************************************/ + + /** + * Internal dispatch routine + */ + template < + typename ScanInitKernelPtr, ///< Function type of cub::ScanInitKernel + typename ScanKernelPtr, ///< Function type of cub::ScanKernel + typename InputIteratorRA, ///< Random-access iterator type for input (may be a simple pointer type) + typename OutputIteratorRA, ///< Random-access iterator type for output (may be a simple pointer type) + typename ScanOp, ///< Binary scan operator type having member T operator()(const T &a, const T &b) + typename Identity, ///< Identity value type (cub::NullType for inclusive scans) + typename SizeT> ///< Integer type used for global array indexing + __host__ __device__ __forceinline__ + static cudaError_t Dispatch( + int ptx_version, ///< [in] PTX version + void *d_temp_storage, ///< [in] %Device allocation of temporary storage. When NULL, the required allocation size is returned in \p temp_storage_bytes and no work is done. + size_t &temp_storage_bytes, ///< [in,out] Size in bytes of \p d_temp_storage allocation. + ScanInitKernelPtr init_kernel, ///< [in] Kernel function pointer to parameterization of cub::ScanInitKernel + ScanKernelPtr scan_kernel, ///< [in] Kernel function pointer to parameterization of cub::ScanKernel + KernelDispachParams &scan_dispatch_params, ///< [in] Dispatch parameters that match the policy that \p scan_kernel was compiled for + InputIteratorRA d_in, ///< [in] Iterator pointing to scan input + OutputIteratorRA d_out, ///< [in] Iterator pointing to scan output + ScanOp scan_op, ///< [in] Binary scan operator + Identity identity, ///< [in] Identity element + SizeT num_items, ///< [in] Total number of items to scan + cudaStream_t stream = 0, ///< [in] [optional] CUDA stream to launch kernels within. Default is stream0. + bool stream_synchronous = false) ///< [in] [optional] Whether or not to synchronize the stream after every kernel launch to check for errors. Default is \p false. + { + +#ifndef CUB_RUNTIME_ENABLED + + // Kernel launch not supported from this device + return CubDebug(cudaErrorNotSupported); + +#else + + enum + { + TILE_STATUS_PADDING = 32, + INIT_KERNEL_THREADS = 128 + }; + + // Data type + typedef typename std::iterator_traits::value_type T; + + // Tile status descriptor type + typedef ScanTileDescriptor ScanTileDescriptorT; + + cudaError error = cudaSuccess; + do + { + // Number of input tiles + int num_tiles = (num_items + scan_dispatch_params.tile_size - 1) / scan_dispatch_params.tile_size; + + // Temporary storage allocation requirements + void* allocations[2]; + size_t allocation_sizes[2] = + { + (num_tiles + TILE_STATUS_PADDING) * sizeof(ScanTileDescriptorT), // bytes needed for tile status descriptors + GridQueue::AllocationSize() // bytes needed for grid queue descriptor + }; + + // Alias temporaries (or set the necessary size of the storage allocation) + if (CubDebug(error = AliasTemporaries(d_temp_storage, temp_storage_bytes, allocations, allocation_sizes))) break; + + // Return if the caller is simply requesting the size of the storage allocation + if (d_temp_storage == NULL) + return cudaSuccess; + + // Global list of tile status + ScanTileDescriptorT *d_tile_status = (ScanTileDescriptorT*) allocations[0]; + + // Grid queue descriptor + GridQueue queue(allocations[1]); + + // Log init_kernel configuration + int init_grid_size = (num_tiles + INIT_KERNEL_THREADS - 1) / INIT_KERNEL_THREADS; + if (stream_synchronous) CubLog("Invoking init_kernel<<<%d, %d, 0, %lld>>>()\n", init_grid_size, INIT_KERNEL_THREADS, (long long) stream); + + // Invoke init_kernel to initialize tile descriptors and queue descriptors + init_kernel<<>>( + queue, + d_tile_status, + num_tiles); + + // Sync the stream if specified + if (stream_synchronous && (CubDebug(error = SyncStream(stream)))) break; + + // Get grid size for multi-block kernel + int scan_grid_size; + int multi_sm_occupancy = -1; + if (ptx_version < 200) + { + // We don't have atomics (or don't have fast ones), so just assign one + // block per tile (limited to 65K tiles) + scan_grid_size = num_tiles; + } + else + { + // We have atomics and can thus reuse blocks across multiple tiles using a queue descriptor. + // Get GPU id + int device_ordinal; + if (CubDebug(error = cudaGetDevice(&device_ordinal))) break; + + // Get SM count + int sm_count; + if (CubDebug(error = cudaDeviceGetAttribute (&sm_count, cudaDevAttrMultiProcessorCount, device_ordinal))) break; + + // Get a rough estimate of scan_kernel SM occupancy based upon the maximum SM occupancy of the targeted PTX architecture + multi_sm_occupancy = CUB_MIN( + ArchProps::MAX_SM_THREADBLOCKS, + ArchProps::MAX_SM_THREADS / scan_dispatch_params.block_threads); + +#ifndef __CUDA_ARCH__ + // We're on the host, so come up with a + Device device_props; + if (CubDebug(error = device_props.Init(device_ordinal))) break; + + if (CubDebug(error = device_props.MaxSmOccupancy( + multi_sm_occupancy, + scan_kernel, + scan_dispatch_params.block_threads))) break; +#endif + // Get device occupancy for scan_kernel + int scan_occupancy = multi_sm_occupancy * sm_count; + + // Get grid size for scan_kernel + scan_grid_size = (num_tiles < scan_occupancy) ? + num_tiles : // Not enough to fill the device with threadblocks + scan_occupancy; // Fill the device with threadblocks + } + + // Log scan_kernel configuration + if (stream_synchronous) CubLog("Invoking scan_kernel<<<%d, %d, 0, %lld>>>(), %d items per thread, %d SM occupancy\n", + scan_grid_size, scan_dispatch_params.block_threads, (long long) stream, scan_dispatch_params.items_per_thread, multi_sm_occupancy); + + // Invoke scan_kernel + scan_kernel<<>>( + d_in, + d_out, + d_tile_status, + scan_op, + identity, + num_items, + queue); + + // Sync the stream if specified + if (stream_synchronous && (CubDebug(error = SyncStream(stream)))) break; + } + while (0); + + return error; + +#endif // CUB_RUNTIME_ENABLED + } + + + + /** + * Internal scan dispatch routine for using default tuning policies + */ + template < + typename InputIteratorRA, ///< Random-access iterator type for input (may be a simple pointer type) + typename OutputIteratorRA, ///< Random-access iterator type for output (may be a simple pointer type) + typename ScanOp, ///< Binary scan operator type having member T operator()(const T &a, const T &b) + typename Identity, ///< Identity value type (cub::NullType for inclusive scans) + typename SizeT> ///< Integer type used for global array indexing + __host__ __device__ __forceinline__ + static cudaError_t Dispatch( + void *d_temp_storage, ///< [in] %Device allocation of temporary storage. When NULL, the required allocation size is returned in \p temp_storage_bytes and no work is done. + size_t &temp_storage_bytes, ///< [in,out] Size in bytes of \p d_temp_storage allocation. + InputIteratorRA d_in, ///< [in] Iterator pointing to scan input + OutputIteratorRA d_out, ///< [in] Iterator pointing to scan output + ScanOp scan_op, ///< [in] Binary scan operator + Identity identity, ///< [in] Identity element + SizeT num_items, ///< [in] Total number of items to scan + cudaStream_t stream = 0, ///< [in] [optional] CUDA stream to launch kernels within. Default is stream0. + bool stream_synchronous = false) ///< [in] [optional] Whether or not to synchronize the stream after every kernel launch to check for errors. Default is \p false. + { + // Data type + typedef typename std::iterator_traits::value_type T; + + // Tuning polices + typedef PtxDefaultPolicies PtxDefaultPolicies; // Wrapper of default kernel policies + typedef typename PtxDefaultPolicies::ScanPolicy ScanPolicy; // Scan kernel policy + + cudaError error = cudaSuccess; + do + { + // Declare dispatch parameters + KernelDispachParams scan_dispatch_params; + + int ptx_version; +#ifdef __CUDA_ARCH__ + // We're on the device, so initialize the dispatch parameters with the PtxDefaultPolicies directly + scan_dispatch_params.Init(); + ptx_version = CUB_PTX_ARCH; +#else + // We're on the host, so lookup and initialize the dispatch parameters with the policies that match the device's PTX version + if (CubDebug(error = PtxVersion(ptx_version))) break; + PtxDefaultPolicies::InitDispatchParams(ptx_version, scan_dispatch_params); +#endif + + Dispatch( + ptx_version, + d_temp_storage, + temp_storage_bytes, + ScanInitKernel, + ScanKernel, + scan_dispatch_params, + d_in, + d_out, + scan_op, + identity, + num_items, + stream, + stream_synchronous); + + if (CubDebug(error)) break; + } + while (0); + + return error; + } + + #endif // DOXYGEN_SHOULD_SKIP_THIS + + + /******************************************************************//** + * \name Exclusive scans + *********************************************************************/ + //@{ + + /** + * \brief Computes a device-wide exclusive prefix sum. + * + * \devicestorage + * + * \cdp + * + * \iterator + * + * \par + * The code snippet below illustrates the exclusive prefix sum of a device vector of \p int items. + * \par + * \code + * #include + * ... + * + * // Declare and initialize device pointers for input and output + * int *d_scan_input, *d_scan_output; + * int num_items = ... + * + * ... + * + * // Determine temporary device storage requirements for exclusive prefix sum + * void *d_temp_storage = NULL; + * size_t temp_storage_bytes = 0; + * cub::DeviceScan::ExclusiveSum(d_temp_storage, temp_storage_bytes, d_scan_input, d_scan_output, num_items); + * + * // Allocate temporary storage for exclusive prefix sum + * cudaMalloc(&d_temp_storage, temp_storage_bytes); + * + * // Run exclusive prefix sum + * cub::DeviceScan::ExclusiveSum(d_temp_storage, temp_storage_bytes, d_scan_input, d_scan_output, num_items); + * + * \endcode + * + * \tparam InputIteratorRA [inferred] Random-access iterator type for input (may be a simple pointer type) + * \tparam OutputIteratorRA [inferred] Random-access iterator type for output (may be a simple pointer type) + */ + template < + typename InputIteratorRA, + typename OutputIteratorRA> + __host__ __device__ __forceinline__ + static cudaError_t ExclusiveSum( + void *d_temp_storage, ///< [in] %Device allocation of temporary storage. When NULL, the required allocation size is returned in \p temp_storage_bytes and no work is done. + size_t &temp_storage_bytes, ///< [in,out] Size in bytes of \p d_temp_storage allocation. + InputIteratorRA d_in, ///< [in] Iterator pointing to scan input + OutputIteratorRA d_out, ///< [in] Iterator pointing to scan output + int num_items, ///< [in] Total number of items to scan + cudaStream_t stream = 0, ///< [in] [optional] CUDA stream to launch kernels within. Default is stream0. + bool stream_synchronous = false) ///< [in] [optional] Whether or not to synchronize the stream after every kernel launch to check for errors. May cause significant slowdown. Default is \p false. + { + typedef typename std::iterator_traits::value_type T; + return Dispatch(d_temp_storage, temp_storage_bytes, d_in, d_out, Sum(), T(), num_items, stream, stream_synchronous); + } + + + /** + * \brief Computes a device-wide exclusive prefix scan using the specified binary \p scan_op functor. + * + * \par + * Supports non-commutative scan operators. + * + * \devicestorage + * + * \cdp + * + * \iterator + * + * \par + * The code snippet below illustrates the exclusive prefix scan of a device vector of \p int items. + * \par + * \code + * #include + * ... + * + * // Declare and initialize device pointers for input and output + * int *d_scan_input, *d_scan_output; + * int num_items = ... + * + * ... + * + * // Determine temporary device storage requirements for exclusive prefix scan + * void *d_temp_storage = NULL; + * size_t temp_storage_bytes = 0; + * cub::DeviceScan::ExclusiveScan(d_temp_storage, temp_storage_bytes, d_scan_input, d_scan_output, cub::Max(), (int) MIN_INT, num_items); + * + * // Allocate temporary storage for exclusive prefix scan + * cudaMalloc(&d_temp_storage, temp_storage_bytes); + * + * // Run exclusive prefix scan (max) + * cub::DeviceScan::ExclusiveScan(d_temp_storage, temp_storage_bytes, d_scan_input, d_scan_output, cub::Max(), (int) MIN_INT, num_items); + * + * \endcode + * + * \tparam InputIteratorRA [inferred] Random-access iterator type for input (may be a simple pointer type) + * \tparam OutputIteratorRA [inferred] Random-access iterator type for output (may be a simple pointer type) + * \tparam ScanOp [inferred] Binary scan operator type having member T operator()(const T &a, const T &b) + * \tparam Identity [inferred] Type of the \p identity value used Binary scan operator type having member T operator()(const T &a, const T &b) + */ + template < + typename InputIteratorRA, + typename OutputIteratorRA, + typename ScanOp, + typename Identity> + __host__ __device__ __forceinline__ + static cudaError_t ExclusiveScan( + void *d_temp_storage, ///< [in] %Device allocation of temporary storage. When NULL, the required allocation size is returned in \p temp_storage_bytes and no work is done. + size_t &temp_storage_bytes, ///< [in,out] Size in bytes of \p d_temp_storage allocation. + InputIteratorRA d_in, ///< [in] Iterator pointing to scan input + OutputIteratorRA d_out, ///< [in] Iterator pointing to scan output + ScanOp scan_op, ///< [in] Binary scan operator + Identity identity, ///< [in] Identity element + int num_items, ///< [in] Total number of items to scan + cudaStream_t stream = 0, ///< [in] [optional] CUDA stream to launch kernels within. Default is stream0. + bool stream_synchronous = false) ///< [in] [optional] Whether or not to synchronize the stream after every kernel launch to check for errors. May cause significant slowdown. Default is \p false. + { + return Dispatch(d_temp_storage, temp_storage_bytes, d_in, d_out, scan_op, identity, num_items, stream, stream_synchronous); + } + + + //@} end member group + /******************************************************************//** + * \name Inclusive scans + *********************************************************************/ + //@{ + + + /** + * \brief Computes a device-wide inclusive prefix sum. + * + * \devicestorage + * + * \cdp + * + * \iterator + * + * \par + * The code snippet below illustrates the inclusive prefix sum of a device vector of \p int items. + * \par + * \code + * #include + * ... + * + * // Declare and initialize device pointers for input and output + * int *d_scan_input, *d_scan_output; + * int num_items = ... + * ... + * + * // Determine temporary device storage requirements for inclusive prefix sum + * void *d_temp_storage = NULL; + * size_t temp_storage_bytes = 0; + * cub::DeviceScan::InclusiveSum(d_temp_storage, temp_storage_bytes, d_scan_input, d_scan_output, num_items); + * + * // Allocate temporary storage for inclusive prefix sum + * cudaMalloc(&d_temp_storage, temp_storage_bytes); + * + * // Run inclusive prefix sum + * cub::DeviceScan::InclusiveSum(d_temp_storage, temp_storage_bytes, d_scan_input, d_scan_output, num_items); + * + * \endcode + * + * \tparam InputIteratorRA [inferred] Random-access iterator type for input (may be a simple pointer type) + * \tparam OutputIteratorRA [inferred] Random-access iterator type for output (may be a simple pointer type) + */ + template < + typename InputIteratorRA, + typename OutputIteratorRA> + __host__ __device__ __forceinline__ + static cudaError_t InclusiveSum( + void *d_temp_storage, ///< [in] %Device allocation of temporary storage. When NULL, the required allocation size is returned in \p temp_storage_bytes and no work is done. + size_t &temp_storage_bytes, ///< [in,out] Size in bytes of \p d_temp_storage allocation. + InputIteratorRA d_in, ///< [in] Iterator pointing to scan input + OutputIteratorRA d_out, ///< [in] Iterator pointing to scan output + int num_items, ///< [in] Total number of items to scan + cudaStream_t stream = 0, ///< [in] [optional] CUDA stream to launch kernels within. Default is stream0. + bool stream_synchronous = false) ///< [in] [optional] Whether or not to synchronize the stream after every kernel launch to check for errors. May cause significant slowdown. Default is \p false. + { + return Dispatch(d_temp_storage, temp_storage_bytes, d_in, d_out, Sum(), NullType(), num_items, stream, stream_synchronous); + } + + + /** + * \brief Computes a device-wide inclusive prefix scan using the specified binary \p scan_op functor. + * + * \par + * Supports non-commutative scan operators. + * + * \devicestorage + * + * \cdp + * + * \iterator + * + * \par + * The code snippet below illustrates the inclusive prefix scan of a device vector of \p int items. + * \par + * \code + * #include + * ... + * + * // Declare and initialize device pointers for input and output + * int *d_scan_input, *d_scan_output; + * int num_items = ... + * ... + * + * // Determine temporary device storage requirements for inclusive prefix scan + * void *d_temp_storage = NULL; + * size_t temp_storage_bytes = 0; + * cub::DeviceScan::InclusiveScan(d_temp_storage, temp_storage_bytes, d_scan_input, d_scan_output, cub::Max(), num_items); + * + * // Allocate temporary storage for inclusive prefix scan + * cudaMalloc(&d_temp_storage, temp_storage_bytes); + * + * // Run inclusive prefix scan (max) + * cub::DeviceScan::InclusiveScan(d_temp_storage, temp_storage_bytes, d_scan_input, d_scan_output, cub::Max(), num_items); + * + * \endcode + * + * \tparam InputIteratorRA [inferred] Random-access iterator type for input (may be a simple pointer type) + * \tparam OutputIteratorRA [inferred] Random-access iterator type for output (may be a simple pointer type) + * \tparam ScanOp [inferred] Binary scan operator type having member T operator()(const T &a, const T &b) + */ + template < + typename InputIteratorRA, + typename OutputIteratorRA, + typename ScanOp> + __host__ __device__ __forceinline__ + static cudaError_t InclusiveScan( + void *d_temp_storage, ///< [in] %Device allocation of temporary storage. When NULL, the required allocation size is returned in \p temp_storage_bytes and no work is done. + size_t &temp_storage_bytes, ///< [in,out] Size in bytes of \p d_temp_storage allocation. + InputIteratorRA d_in, ///< [in] Iterator pointing to scan input + OutputIteratorRA d_out, ///< [in] Iterator pointing to scan output + ScanOp scan_op, ///< [in] Binary scan operator + int num_items, ///< [in] Total number of items to scan + cudaStream_t stream = 0, ///< [in] [optional] CUDA stream to launch kernels within. Default is stream0. + bool stream_synchronous = false) ///< [in] [optional] Whether or not to synchronize the stream after every kernel launch to check for errors. May cause significant slowdown. Default is \p false. + { + return Dispatch(d_temp_storage, temp_storage_bytes, d_in, d_out, scan_op, NullType(), num_items, stream, stream_synchronous); + } + +}; + + +} // CUB namespace +CUB_NS_POSTFIX // Optional outer namespace(s) + + diff --git a/lib/kokkos/TPL/cub/grid/grid_barrier.cuh b/lib/kokkos/TPL/cub/grid/grid_barrier.cuh new file mode 100755 index 0000000000..ebdc4b552a --- /dev/null +++ b/lib/kokkos/TPL/cub/grid/grid_barrier.cuh @@ -0,0 +1,211 @@ +/****************************************************************************** + * Copyright (c) 2011, Duane Merrill. All rights reserved. + * Copyright (c) 2011-2013, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the NVIDIA CORPORATION nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + ******************************************************************************/ + +/** + * \file + * cub::GridBarrier implements a software global barrier among thread blocks within a CUDA grid + */ + +#pragma once + +#include "../util_debug.cuh" +#include "../util_namespace.cuh" +#include "../thread/thread_load.cuh" + +/// Optional outer namespace(s) +CUB_NS_PREFIX + +/// CUB namespace +namespace cub { + + +/** + * \addtogroup GridModule + * @{ + */ + + +/** + * \brief GridBarrier implements a software global barrier among thread blocks within a CUDA grid + */ +class GridBarrier +{ +protected : + + typedef unsigned int SyncFlag; + + // Counters in global device memory + SyncFlag* d_sync; + +public: + + /** + * Constructor + */ + GridBarrier() : d_sync(NULL) {} + + + /** + * Synchronize + */ + __device__ __forceinline__ void Sync() const + { + volatile SyncFlag *d_vol_sync = d_sync; + + // Threadfence and syncthreads to make sure global writes are visible before + // thread-0 reports in with its sync counter + __threadfence(); + __syncthreads(); + + if (blockIdx.x == 0) + { + // Report in ourselves + if (threadIdx.x == 0) + { + d_vol_sync[blockIdx.x] = 1; + } + + __syncthreads(); + + // Wait for everyone else to report in + for (int peer_block = threadIdx.x; peer_block < gridDim.x; peer_block += blockDim.x) + { + while (ThreadLoad(d_sync + peer_block) == 0) + { + __threadfence_block(); + } + } + + __syncthreads(); + + // Let everyone know it's safe to proceed + for (int peer_block = threadIdx.x; peer_block < gridDim.x; peer_block += blockDim.x) + { + d_vol_sync[peer_block] = 0; + } + } + else + { + if (threadIdx.x == 0) + { + // Report in + d_vol_sync[blockIdx.x] = 1; + + // Wait for acknowledgment + while (ThreadLoad(d_sync + blockIdx.x) == 1) + { + __threadfence_block(); + } + } + + __syncthreads(); + } + } +}; + + +/** + * \brief GridBarrierLifetime extends GridBarrier to provide lifetime management of the temporary device storage needed for cooperation. + * + * Uses RAII for lifetime, i.e., device resources are reclaimed when + * the destructor is called. + */ +class GridBarrierLifetime : public GridBarrier +{ +protected: + + // Number of bytes backed by d_sync + size_t sync_bytes; + +public: + + /** + * Constructor + */ + GridBarrierLifetime() : GridBarrier(), sync_bytes(0) {} + + + /** + * DeviceFrees and resets the progress counters + */ + cudaError_t HostReset() + { + cudaError_t retval = cudaSuccess; + if (d_sync) + { + CubDebug(retval = cudaFree(d_sync)); + d_sync = NULL; + } + sync_bytes = 0; + return retval; + } + + + /** + * Destructor + */ + virtual ~GridBarrierLifetime() + { + HostReset(); + } + + + /** + * Sets up the progress counters for the next kernel launch (lazily + * allocating and initializing them if necessary) + */ + cudaError_t Setup(int sweep_grid_size) + { + cudaError_t retval = cudaSuccess; + do { + size_t new_sync_bytes = sweep_grid_size * sizeof(SyncFlag); + if (new_sync_bytes > sync_bytes) + { + if (d_sync) + { + if (CubDebug(retval = cudaFree(d_sync))) break; + } + + sync_bytes = new_sync_bytes; + + // Allocate and initialize to zero + if (CubDebug(retval = cudaMalloc((void**) &d_sync, sync_bytes))) break; + if (CubDebug(retval = cudaMemset(d_sync, 0, new_sync_bytes))) break; + } + } while (0); + + return retval; + } +}; + + +/** @} */ // end group GridModule + +} // CUB namespace +CUB_NS_POSTFIX // Optional outer namespace(s) + diff --git a/lib/kokkos/TPL/cub/grid/grid_even_share.cuh b/lib/kokkos/TPL/cub/grid/grid_even_share.cuh new file mode 100755 index 0000000000..defe9e0a66 --- /dev/null +++ b/lib/kokkos/TPL/cub/grid/grid_even_share.cuh @@ -0,0 +1,197 @@ +/****************************************************************************** + * Copyright (c) 2011, Duane Merrill. All rights reserved. + * Copyright (c) 2011-2013, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the NVIDIA CORPORATION nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + ******************************************************************************/ + +/** + * \file + * cub::GridEvenShare is a descriptor utility for distributing input among CUDA threadblocks in an "even-share" fashion. Each threadblock gets roughly the same number of fixed-size work units (grains). + */ + + +#pragma once + +#include "../util_namespace.cuh" +#include "../util_macro.cuh" + +/// Optional outer namespace(s) +CUB_NS_PREFIX + +/// CUB namespace +namespace cub { + + +/** + * \addtogroup GridModule + * @{ + */ + + +/** + * \brief GridEvenShare is a descriptor utility for distributing input among CUDA threadblocks in an "even-share" fashion. Each threadblock gets roughly the same number of fixed-size work units (grains). + * + * \par Overview + * GridEvenShare indicates which sections of input are to be mapped onto which threadblocks. + * Threadblocks may receive one of three different amounts of work: "big", "normal", + * and "last". The "big" workloads are one scheduling grain larger than "normal". The "last" work unit + * for the last threadblock may be partially-full if the input is not an even multiple of + * the scheduling grain size. + * + * \par + * Before invoking a child grid, a parent thread will typically construct and initialize an instance of + * GridEvenShare using \p GridInit(). The instance can be passed to child threadblocks which can + * initialize their per-threadblock offsets using \p BlockInit(). + * + * \tparam SizeT Integer type for array indexing + */ +template +class GridEvenShare +{ +private: + + SizeT total_grains; + int big_blocks; + SizeT big_share; + SizeT normal_share; + SizeT normal_base_offset; + + +public: + + /// Total number of input items + SizeT num_items; + + /// Grid size in threadblocks + int grid_size; + + /// Offset into input marking the beginning of the owning thread block's segment of input tiles + SizeT block_offset; + + /// Offset into input of marking the end (one-past) of the owning thread block's segment of input tiles + SizeT block_oob; + + /** + * \brief Block-based constructor for single-block grids. + */ + __device__ __forceinline__ GridEvenShare(SizeT num_items) : + num_items(num_items), + grid_size(1), + block_offset(0), + block_oob(num_items) {} + + + /** + * \brief Default constructor. Zero-initializes block-specific fields. + */ + __host__ __device__ __forceinline__ GridEvenShare() : + num_items(0), + grid_size(0), + block_offset(0), + block_oob(0) {} + + + /** + * \brief Initializes the grid-specific members \p num_items and \p grid_size. To be called prior prior to kernel launch) + */ + __host__ __device__ __forceinline__ void GridInit( + SizeT num_items, ///< Total number of input items + int max_grid_size, ///< Maximum grid size allowable (actual grid size may be less if not warranted by the the number of input items) + int schedule_granularity) ///< Granularity by which the input can be parcelled into and distributed among threablocks. Usually the thread block's native tile size (or a multiple thereof. + { + this->num_items = num_items; + this->block_offset = 0; + this->block_oob = 0; + this->total_grains = (num_items + schedule_granularity - 1) / schedule_granularity; + this->grid_size = CUB_MIN(total_grains, max_grid_size); + SizeT grains_per_block = total_grains / grid_size; + this->big_blocks = total_grains - (grains_per_block * grid_size); // leftover grains go to big blocks + this->normal_share = grains_per_block * schedule_granularity; + this->normal_base_offset = big_blocks * schedule_granularity; + this->big_share = normal_share + schedule_granularity; + } + + + /** + * \brief Initializes the threadblock-specific details (e.g., to be called by each threadblock after startup) + */ + __device__ __forceinline__ void BlockInit() + { + if (blockIdx.x < big_blocks) + { + // This threadblock gets a big share of grains (grains_per_block + 1) + block_offset = (blockIdx.x * big_share); + block_oob = block_offset + big_share; + } + else if (blockIdx.x < total_grains) + { + // This threadblock gets a normal share of grains (grains_per_block) + block_offset = normal_base_offset + (blockIdx.x * normal_share); + block_oob = block_offset + normal_share; + } + + // Last threadblock + if (blockIdx.x == grid_size - 1) + { + block_oob = num_items; + } + } + + + /** + * Print to stdout + */ + __host__ __device__ __forceinline__ void Print() + { + printf( +#ifdef __CUDA_ARCH__ + "\tthreadblock(%d) " + "block_offset(%lu) " + "block_oob(%lu) " +#endif + "num_items(%lu) " + "total_grains(%lu) " + "big_blocks(%lu) " + "big_share(%lu) " + "normal_share(%lu)\n", +#ifdef __CUDA_ARCH__ + blockIdx.x, + (unsigned long) block_offset, + (unsigned long) block_oob, +#endif + (unsigned long) num_items, + (unsigned long) total_grains, + (unsigned long) big_blocks, + (unsigned long) big_share, + (unsigned long) normal_share); + } +}; + + + +/** @} */ // end group GridModule + +} // CUB namespace +CUB_NS_POSTFIX // Optional outer namespace(s) diff --git a/lib/kokkos/TPL/cub/grid/grid_mapping.cuh b/lib/kokkos/TPL/cub/grid/grid_mapping.cuh new file mode 100755 index 0000000000..419f9ac0e0 --- /dev/null +++ b/lib/kokkos/TPL/cub/grid/grid_mapping.cuh @@ -0,0 +1,95 @@ +/****************************************************************************** + * Copyright (c) 2011, Duane Merrill. All rights reserved. + * Copyright (c) 2011-2013, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the NVIDIA CORPORATION nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + ******************************************************************************/ + +/** + * \file + * cub::GridMappingStrategy enumerates alternative strategies for mapping constant-sized tiles of device-wide data onto a grid of CUDA thread blocks. + */ + +#pragma once + +#include "../util_namespace.cuh" + +/// Optional outer namespace(s) +CUB_NS_PREFIX + +/// CUB namespace +namespace cub { + + +/** + * \addtogroup GridModule + * @{ + */ + + +/****************************************************************************** + * Mapping policies + *****************************************************************************/ + + +/** + * \brief cub::GridMappingStrategy enumerates alternative strategies for mapping constant-sized tiles of device-wide data onto a grid of CUDA thread blocks. + */ +enum GridMappingStrategy +{ + /** + * \brief An "even-share" strategy for assigning input tiles to thread blocks. + * + * \par Overview + * The input is evenly partitioned into \p p segments, where \p p is + * constant and corresponds loosely to the number of thread blocks that may + * actively reside on the target device. Each segment is comprised of + * consecutive tiles, where a tile is a small, constant-sized unit of input + * to be processed to completion before the thread block terminates or + * obtains more work. The kernel invokes \p p thread blocks, each + * of which iteratively consumes a segment of n/p elements + * in tile-size increments. + */ + GRID_MAPPING_EVEN_SHARE, + + /** + * \brief A dynamic "queue-based" strategy for assigning input tiles to thread blocks. + * + * \par Overview + * The input is treated as a queue to be dynamically consumed by a grid of + * thread blocks. Work is atomically dequeued in tiles, where a tile is a + * unit of input to be processed to completion before the thread block + * terminates or obtains more work. The grid size \p p is constant, + * loosely corresponding to the number of thread blocks that may actively + * reside on the target device. + */ + GRID_MAPPING_DYNAMIC, +}; + + +/** @} */ // end group GridModule + +} // CUB namespace +CUB_NS_POSTFIX // Optional outer namespace(s) + diff --git a/lib/kokkos/TPL/cub/grid/grid_queue.cuh b/lib/kokkos/TPL/cub/grid/grid_queue.cuh new file mode 100755 index 0000000000..009260d87b --- /dev/null +++ b/lib/kokkos/TPL/cub/grid/grid_queue.cuh @@ -0,0 +1,207 @@ +/****************************************************************************** + * Copyright (c) 2011, Duane Merrill. All rights reserved. + * Copyright (c) 2011-2013, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the NVIDIA CORPORATION nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + ******************************************************************************/ + +/** + * \file + * cub::GridQueue is a descriptor utility for dynamic queue management. + */ + +#pragma once + +#include "../util_namespace.cuh" +#include "../util_debug.cuh" + +/// Optional outer namespace(s) +CUB_NS_PREFIX + +/// CUB namespace +namespace cub { + + +/** + * \addtogroup GridModule + * @{ + */ + + +/** + * \brief GridQueue is a descriptor utility for dynamic queue management. + * + * \par Overview + * GridQueue descriptors provides abstractions for "filling" or + * "draining" globally-shared vectors. + * + * \par + * A "filling" GridQueue works by atomically-adding to a zero-initialized counter, + * returning a unique offset for the calling thread to write its items. + * The GridQueue maintains the total "fill-size". The fill counter must be reset + * using GridQueue::ResetFill by the host or kernel instance prior to the kernel instance that + * will be filling. + * + * \par + * Similarly a "draining" GridQueue works by works by atomically-incrementing a + * zero-initialized counter, returning a unique offset for the calling thread to + * read its items. Threads can safely drain until the array's logical fill-size is + * exceeded. The drain counter must be reset using GridQueue::ResetDrain or + * GridQueue::ResetDrainAfterFill by the host or kernel instance prior to the kernel instance that + * will be filling. (For dynamic work distribution of existing data, the corresponding fill-size + * is simply the number of elements in the array.) + * + * \par + * Iterative work management can be implemented simply with a pair of flip-flopping + * work buffers, each with an associated set of fill and drain GridQueue descriptors. + * + * \tparam SizeT Integer type for array indexing + */ +template +class GridQueue +{ +private: + + /// Counter indices + enum + { + FILL = 0, + DRAIN = 1, + }; + + /// Pair of counters + SizeT *d_counters; + +public: + + /// Returns the device allocation size in bytes needed to construct a GridQueue instance + __host__ __device__ __forceinline__ + static size_t AllocationSize() + { + return sizeof(SizeT) * 2; + } + + + /// Constructs an invalid GridQueue descriptor around the device storage allocation + __host__ __device__ __forceinline__ GridQueue( + void *d_storage) ///< Device allocation to back the GridQueue. Must be at least as big as AllocationSize(). + : + d_counters((SizeT*) d_storage) + {} + + + /// This operation resets the drain so that it may advance to meet the existing fill-size. To be called by the host or by a kernel prior to that which will be draining. + __host__ __device__ __forceinline__ cudaError_t ResetDrainAfterFill(cudaStream_t stream = 0) + { +#ifdef __CUDA_ARCH__ + d_counters[DRAIN] = 0; + return cudaSuccess; +#else + return ResetDrain(0, stream); +#endif + } + + /// This operation sets the fill-size and resets the drain counter, preparing the GridQueue for draining in the next kernel instance. To be called by the host or by a kernel prior to that which will be draining. + __host__ __device__ __forceinline__ cudaError_t ResetDrain( + SizeT fill_size, + cudaStream_t stream = 0) + { +#ifdef __CUDA_ARCH__ + d_counters[FILL] = fill_size; + d_counters[DRAIN] = 0; + return cudaSuccess; +#else + SizeT counters[2]; + counters[FILL] = fill_size; + counters[DRAIN] = 0; + return CubDebug(cudaMemcpyAsync(d_counters, counters, sizeof(SizeT) * 2, cudaMemcpyHostToDevice, stream)); +#endif + } + + + /// This operation resets the fill counter. To be called by the host or by a kernel prior to that which will be filling. + __host__ __device__ __forceinline__ cudaError_t ResetFill() + { +#ifdef __CUDA_ARCH__ + d_counters[FILL] = 0; + return cudaSuccess; +#else + return CubDebug(cudaMemset(d_counters + FILL, 0, sizeof(SizeT))); +#endif + } + + + /// Returns the fill-size established by the parent or by the previous kernel. + __host__ __device__ __forceinline__ cudaError_t FillSize( + SizeT &fill_size, + cudaStream_t stream = 0) + { +#ifdef __CUDA_ARCH__ + fill_size = d_counters[FILL]; +#else + return CubDebug(cudaMemcpyAsync(&fill_size, d_counters + FILL, sizeof(SizeT), cudaMemcpyDeviceToHost, stream)); +#endif + } + + + /// Drain num_items. Returns offset from which to read items. + __device__ __forceinline__ SizeT Drain(SizeT num_items) + { + return atomicAdd(d_counters + DRAIN, num_items); + } + + + /// Fill num_items. Returns offset from which to write items. + __device__ __forceinline__ SizeT Fill(SizeT num_items) + { + return atomicAdd(d_counters + FILL, num_items); + } +}; + + +#ifndef DOXYGEN_SHOULD_SKIP_THIS // Do not document + + +/** + * Reset grid queue (call with 1 block of 1 thread) + */ +template +__global__ void ResetDrainKernel( + GridQueue grid_queue, + SizeT num_items) +{ + grid_queue.ResetDrain(num_items); +} + + + +#endif // DOXYGEN_SHOULD_SKIP_THIS + + +/** @} */ // end group GridModule + +} // CUB namespace +CUB_NS_POSTFIX // Optional outer namespace(s) + + diff --git a/lib/kokkos/TPL/cub/host/spinlock.cuh b/lib/kokkos/TPL/cub/host/spinlock.cuh new file mode 100755 index 0000000000..5621b6f1a3 --- /dev/null +++ b/lib/kokkos/TPL/cub/host/spinlock.cuh @@ -0,0 +1,123 @@ +/****************************************************************************** + * Copyright (c) 2011, Duane Merrill. All rights reserved. + * Copyright (c) 2011-2013, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the NVIDIA CORPORATION nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + ******************************************************************************/ + +/** + * \file + * Simple x86/x64 atomic spinlock, portable across MS Windows (cl.exe) & Linux (g++) + */ + + +#pragma once + +#if defined(_WIN32) || defined(_WIN64) + #include + #include + #undef small // Windows is terrible for polluting macro namespace + + /** + * Compiler read/write barrier + */ + #pragma intrinsic(_ReadWriteBarrier) + +#endif + +#include "../util_namespace.cuh" + +/// Optional outer namespace(s) +CUB_NS_PREFIX + +/// CUB namespace +namespace cub { + + +#if defined(_MSC_VER) + + // Microsoft VC++ + typedef long Spinlock; + +#else + + // GNU g++ + typedef int Spinlock; + + /** + * Compiler read/write barrier + */ + __forceinline__ void _ReadWriteBarrier() + { + __sync_synchronize(); + } + + /** + * Atomic exchange + */ + __forceinline__ long _InterlockedExchange(volatile int * const Target, const int Value) + { + // NOTE: __sync_lock_test_and_set would be an acquire barrier, so we force a full barrier + _ReadWriteBarrier(); + return __sync_lock_test_and_set(Target, Value); + } + + /** + * Pause instruction to prevent excess processor bus usage + */ + __forceinline__ void YieldProcessor() + { +#ifndef __arm__ + asm volatile("pause\n": : :"memory"); +#endif // __arm__ + } + +#endif // defined(_MSC_VER) + +/** + * Return when the specified spinlock has been acquired + */ +__forceinline__ void Lock(volatile Spinlock *lock) +{ + while (1) + { + if (!_InterlockedExchange(lock, 1)) return; + while (*lock) YieldProcessor(); + } +} + + +/** + * Release the specified spinlock + */ +__forceinline__ void Unlock(volatile Spinlock *lock) +{ + _ReadWriteBarrier(); + *lock = 0; +} + + +} // CUB namespace +CUB_NS_POSTFIX // Optional outer namespace(s) + diff --git a/lib/kokkos/TPL/cub/thread/thread_load.cuh b/lib/kokkos/TPL/cub/thread/thread_load.cuh new file mode 100755 index 0000000000..ee112b9d5c --- /dev/null +++ b/lib/kokkos/TPL/cub/thread/thread_load.cuh @@ -0,0 +1,429 @@ +/****************************************************************************** + * Copyright (c) 2011, Duane Merrill. All rights reserved. + * Copyright (c) 2011-2013, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the NVIDIA CORPORATION nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + ******************************************************************************/ + +/** + * \file + * Thread utilities for reading memory using PTX cache modifiers. + */ + +#pragma once + +#include + +#include + +#include "../util_ptx.cuh" +#include "../util_type.cuh" +#include "../util_namespace.cuh" + +/// Optional outer namespace(s) +CUB_NS_PREFIX + +/// CUB namespace +namespace cub { + +/** + * \addtogroup IoModule + * @{ + */ + +//----------------------------------------------------------------------------- +// Tags and constants +//----------------------------------------------------------------------------- + +/** + * \brief Enumeration of PTX cache-modifiers for memory load operations. + */ +enum PtxLoadModifier +{ + LOAD_DEFAULT, ///< Default (no modifier) + LOAD_CA, ///< Cache at all levels + LOAD_CG, ///< Cache at global level + LOAD_CS, ///< Cache streaming (likely to be accessed once) + LOAD_CV, ///< Cache as volatile (including cached system lines) + LOAD_LDG, ///< Cache as texture + LOAD_VOLATILE, ///< Volatile (any memory space) +}; + + +/** + * \name Simple I/O + * @{ + */ + +/** + * \brief Thread utility for reading memory using cub::PtxLoadModifier cache modifiers. + * + * Cache modifiers will only be effected for built-in types (i.e., C++ + * primitives and CUDA vector-types). + * + * For example: + * \par + * \code + * #include + * + * // 32-bit load using cache-global modifier: + * int *d_in; + * int val = cub::ThreadLoad(d_in + threadIdx.x); + * + * // 16-bit load using default modifier + * short *d_in; + * short val = cub::ThreadLoad(d_in + threadIdx.x); + * + * // 256-bit load using cache-volatile modifier + * double4 *d_in; + * double4 val = cub::ThreadLoad(d_in + threadIdx.x); + * + * // 96-bit load using default cache modifier (ignoring LOAD_CS) + * struct TestFoo { bool a; short b; }; + * TestFoo *d_struct; + * TestFoo val = cub::ThreadLoad(d_in + threadIdx.x); + * \endcode + * + */ +template < + PtxLoadModifier MODIFIER, + typename InputIteratorRA> +__device__ __forceinline__ typename std::iterator_traits::value_type ThreadLoad(InputIteratorRA itr); + + +//@} end member group + + + +#ifndef DOXYGEN_SHOULD_SKIP_THIS // Do not document + + +/** + * Define a int4 (16B) ThreadLoad specialization for the given PTX load modifier + */ +#define CUB_LOAD_16(cub_modifier, ptx_modifier) \ + template<> \ + __device__ __forceinline__ int4 ThreadLoad(int4* ptr) \ + { \ + int4 retval; \ + asm volatile ("ld."#ptx_modifier".v4.s32 {%0, %1, %2, %3}, [%4];" : \ + "=r"(retval.x), \ + "=r"(retval.y), \ + "=r"(retval.z), \ + "=r"(retval.w) : \ + _CUB_ASM_PTR_(ptr)); \ + return retval; \ + } \ + template<> \ + __device__ __forceinline__ longlong2 ThreadLoad(longlong2* ptr) \ + { \ + longlong2 retval; \ + asm volatile ("ld."#ptx_modifier".v2.s64 {%0, %1}, [%2];" : \ + "=l"(retval.x), \ + "=l"(retval.y) : \ + _CUB_ASM_PTR_(ptr)); \ + return retval; \ + } + +/** + * Define a int2 (8B) ThreadLoad specialization for the given PTX load modifier + */ +#define CUB_LOAD_8(cub_modifier, ptx_modifier) \ + template<> \ + __device__ __forceinline__ short4 ThreadLoad(short4* ptr) \ + { \ + short4 retval; \ + asm volatile ("ld."#ptx_modifier".v4.s16 {%0, %1, %2, %3}, [%4];" : \ + "=h"(retval.x), \ + "=h"(retval.y), \ + "=h"(retval.z), \ + "=h"(retval.w) : \ + _CUB_ASM_PTR_(ptr)); \ + return retval; \ + } \ + template<> \ + __device__ __forceinline__ int2 ThreadLoad(int2* ptr) \ + { \ + int2 retval; \ + asm volatile ("ld."#ptx_modifier".v2.s32 {%0, %1}, [%2];" : \ + "=r"(retval.x), \ + "=r"(retval.y) : \ + _CUB_ASM_PTR_(ptr)); \ + return retval; \ + } \ + template<> \ + __device__ __forceinline__ long long ThreadLoad(long long* ptr) \ + { \ + long long retval; \ + asm volatile ("ld."#ptx_modifier".s64 %0, [%1];" : \ + "=l"(retval) : \ + _CUB_ASM_PTR_(ptr)); \ + return retval; \ + } + +/** + * Define a int (4B) ThreadLoad specialization for the given PTX load modifier + */ +#define CUB_LOAD_4(cub_modifier, ptx_modifier) \ + template<> \ + __device__ __forceinline__ int ThreadLoad(int* ptr) \ + { \ + int retval; \ + asm volatile ("ld."#ptx_modifier".s32 %0, [%1];" : \ + "=r"(retval) : \ + _CUB_ASM_PTR_(ptr)); \ + return retval; \ + } + + +/** + * Define a short (2B) ThreadLoad specialization for the given PTX load modifier + */ +#define CUB_LOAD_2(cub_modifier, ptx_modifier) \ + template<> \ + __device__ __forceinline__ short ThreadLoad(short* ptr) \ + { \ + short retval; \ + asm volatile ("ld."#ptx_modifier".s16 %0, [%1];" : \ + "=h"(retval) : \ + _CUB_ASM_PTR_(ptr)); \ + return retval; \ + } + + +/** + * Define a char (1B) ThreadLoad specialization for the given PTX load modifier + */ +#define CUB_LOAD_1(cub_modifier, ptx_modifier) \ + template<> \ + __device__ __forceinline__ char ThreadLoad(char* ptr) \ + { \ + short retval; \ + asm volatile ( \ + "{" \ + " .reg .s8 datum;" \ + " ld."#ptx_modifier".s8 datum, [%1];" \ + " cvt.s16.s8 %0, datum;" \ + "}" : \ + "=h"(retval) : \ + _CUB_ASM_PTR_(ptr)); \ + return (char) retval; \ + } + + +/** + * Define powers-of-two ThreadLoad specializations for the given PTX load modifier + */ +#define CUB_LOAD_ALL(cub_modifier, ptx_modifier) \ + CUB_LOAD_16(cub_modifier, ptx_modifier) \ + CUB_LOAD_8(cub_modifier, ptx_modifier) \ + CUB_LOAD_4(cub_modifier, ptx_modifier) \ + CUB_LOAD_2(cub_modifier, ptx_modifier) \ + CUB_LOAD_1(cub_modifier, ptx_modifier) \ + + +/** + * Define ThreadLoad specializations for the various PTX load modifiers + */ +#if CUB_PTX_ARCH >= 200 + CUB_LOAD_ALL(LOAD_CA, ca) + CUB_LOAD_ALL(LOAD_CG, cg) + CUB_LOAD_ALL(LOAD_CS, cs) + CUB_LOAD_ALL(LOAD_CV, cv) +#else + // LOAD_CV on SM10-13 uses "volatile.global" to ensure reads from last level + CUB_LOAD_ALL(LOAD_CV, volatile.global) +#endif +#if CUB_PTX_ARCH >= 350 + CUB_LOAD_ALL(LOAD_LDG, global.nc) +#endif + + +/// Helper structure for templated load iteration (inductive case) +template +struct IterateThreadLoad +{ + template + static __device__ __forceinline__ void Load(T *ptr, T *vals) + { + vals[COUNT] = ThreadLoad(ptr + COUNT); + IterateThreadLoad::Load(ptr, vals); + } +}; + +/// Helper structure for templated load iteration (termination case) +template +struct IterateThreadLoad +{ + template + static __device__ __forceinline__ void Load(T *ptr, T *vals) {} +}; + + + +/** + * Load with LOAD_DEFAULT on iterator types + */ +template +__device__ __forceinline__ typename std::iterator_traits::value_type ThreadLoad( + InputIteratorRA itr, + Int2Type modifier, + Int2Type is_pointer) +{ + return *itr; +} + + +/** + * Load with LOAD_DEFAULT on pointer types + */ +template +__device__ __forceinline__ T ThreadLoad( + T *ptr, + Int2Type modifier, + Int2Type is_pointer) +{ + return *ptr; +} + + +/** + * Load with LOAD_VOLATILE on primitive pointer types + */ +template +__device__ __forceinline__ T ThreadLoadVolatile( + T *ptr, + Int2Type is_primitive) +{ + T retval = *reinterpret_cast(ptr); + +#if (CUB_PTX_ARCH <= 130) + if (sizeof(T) == 1) __threadfence_block(); +#endif + + return retval; +} + + +/** + * Load with LOAD_VOLATILE on non-primitive pointer types + */ +template +__device__ __forceinline__ T ThreadLoadVolatile( + T *ptr, + Int2Type is_primitive) +{ + typedef typename WordAlignment::VolatileWord VolatileWord; // Word type for memcopying + enum { NUM_WORDS = sizeof(T) / sizeof(VolatileWord) }; + + // Memcopy from aliased source into array of uninitialized words + typename WordAlignment::UninitializedVolatileWords words; + + #pragma unroll + for (int i = 0; i < NUM_WORDS; ++i) + words.buf[i] = reinterpret_cast(ptr)[i]; + + // Load from words + return *reinterpret_cast(words.buf); +} + + +/** + * Load with LOAD_VOLATILE on pointer types + */ +template +__device__ __forceinline__ T ThreadLoad( + T *ptr, + Int2Type modifier, + Int2Type is_pointer) +{ + return ThreadLoadVolatile(ptr, Int2Type::PRIMITIVE>()); +} + + +#if (CUB_PTX_ARCH <= 130) + +/** + * Load with LOAD_CG uses LOAD_CV in pre-SM20 PTX to ensure coherent reads when run on newer architectures with L1 + */ +template +__device__ __forceinline__ T ThreadLoad( + T *ptr, + Int2Type modifier, + Int2Type is_pointer) +{ + return ThreadLoad(ptr); +} + +#endif // (CUB_PTX_ARCH <= 130) + + +/** + * Load with arbitrary MODIFIER on pointer types + */ +template +__device__ __forceinline__ T ThreadLoad( + T *ptr, + Int2Type modifier, + Int2Type is_pointer) +{ + typedef typename WordAlignment::DeviceWord DeviceWord; + enum { NUM_WORDS = sizeof(T) / sizeof(DeviceWord) }; + + // Memcopy from aliased source into array of uninitialized words + typename WordAlignment::UninitializedDeviceWords words; + + IterateThreadLoad::Load( + reinterpret_cast(ptr), + words.buf); + + // Load from words + return *reinterpret_cast(words.buf); +} + + +/** + * Generic ThreadLoad definition + */ +template < + PtxLoadModifier MODIFIER, + typename InputIteratorRA> +__device__ __forceinline__ typename std::iterator_traits::value_type ThreadLoad(InputIteratorRA itr) +{ + return ThreadLoad( + itr, + Int2Type(), + Int2Type::VALUE>()); +} + + + +#endif // DOXYGEN_SHOULD_SKIP_THIS + + +/** @} */ // end group IoModule + + +} // CUB namespace +CUB_NS_POSTFIX // Optional outer namespace(s) diff --git a/lib/kokkos/TPL/cub/thread/thread_operators.cuh b/lib/kokkos/TPL/cub/thread/thread_operators.cuh new file mode 100755 index 0000000000..bfb3d7c1b5 --- /dev/null +++ b/lib/kokkos/TPL/cub/thread/thread_operators.cuh @@ -0,0 +1,145 @@ +/****************************************************************************** + * Copyright (c) 2011, Duane Merrill. All rights reserved. + * Copyright (c) 2011-2013, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the NVIDIA CORPORATION nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + ******************************************************************************/ + +/** + * \file + * Simple binary operator functor types + */ + +/****************************************************************************** + * Simple functor operators + ******************************************************************************/ + +#pragma once + +#include "../util_macro.cuh" +#include "../util_namespace.cuh" + +/// Optional outer namespace(s) +CUB_NS_PREFIX + +/// CUB namespace +namespace cub { + + +/** + * \addtogroup ThreadModule + * @{ + */ + +/** + * \brief Default equality functor + */ +struct Equality +{ + /// Boolean equality operator, returns (a == b) + template + __host__ __device__ __forceinline__ bool operator()(const T &a, const T &b) + { + return a == b; + } +}; + + +/** + * \brief Default inequality functor + */ +struct Inequality +{ + /// Boolean inequality operator, returns (a != b) + template + __host__ __device__ __forceinline__ bool operator()(const T &a, const T &b) + { + return a != b; + } +}; + + +/** + * \brief Default sum functor + */ +struct Sum +{ + /// Boolean sum operator, returns a + b + template + __host__ __device__ __forceinline__ T operator()(const T &a, const T &b) + { + return a + b; + } +}; + + +/** + * \brief Default max functor + */ +struct Max +{ + /// Boolean max operator, returns (a > b) ? a : b + template + __host__ __device__ __forceinline__ T operator()(const T &a, const T &b) + { + return CUB_MAX(a, b); + } +}; + + +/** + * \brief Default min functor + */ +struct Min +{ + /// Boolean min operator, returns (a < b) ? a : b + template + __host__ __device__ __forceinline__ T operator()(const T &a, const T &b) + { + return CUB_MIN(a, b); + } +}; + + +/** + * \brief Default cast functor + */ +template +struct Cast +{ + /// Boolean max operator, returns (a > b) ? a : b + template + __host__ __device__ __forceinline__ B operator()(const A &a) + { + return (B) a; + } +}; + + + +/** @} */ // end group ThreadModule + + +} // CUB namespace +CUB_NS_POSTFIX // Optional outer namespace(s) diff --git a/lib/kokkos/TPL/cub/thread/thread_reduce.cuh b/lib/kokkos/TPL/cub/thread/thread_reduce.cuh new file mode 100755 index 0000000000..374fd77ae1 --- /dev/null +++ b/lib/kokkos/TPL/cub/thread/thread_reduce.cuh @@ -0,0 +1,145 @@ +/****************************************************************************** + * Copyright (c) 2011, Duane Merrill. All rights reserved. + * Copyright (c) 2011-2013, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the NVIDIA CORPORATION nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + ******************************************************************************/ + +/** + * \file + * Thread utilities for sequential reduction over statically-sized array types + */ + +#pragma once + +#include "../thread/thread_operators.cuh" +#include "../util_namespace.cuh" + +/// Optional outer namespace(s) +CUB_NS_PREFIX + +/// CUB namespace +namespace cub { + +/** + * \addtogroup ThreadModule + * @{ + */ + +/** + * \name Sequential reduction over statically-sized array types + * @{ + */ + +/** + * \brief Perform a sequential reduction over \p LENGTH elements of the \p input array, seeded with the specified \p prefix. The aggregate is returned. + * + * \tparam LENGTH Length of input array + * \tparam T [inferred] The data type to be reduced. + * \tparam ScanOp [inferred] Binary reduction operator type having member T operator()(const T &a, const T &b) + */ +template < + int LENGTH, + typename T, + typename ReductionOp> +__device__ __forceinline__ T ThreadReduce( + T* input, ///< [in] Input array + ReductionOp reduction_op, ///< [in] Binary reduction operator + T prefix) ///< [in] Prefix to seed reduction with +{ + #pragma unroll + for (int i = 0; i < LENGTH; ++i) + { + prefix = reduction_op(prefix, input[i]); + } + + return prefix; +} + + +/** + * \brief Perform a sequential reduction over \p LENGTH elements of the \p input array. The aggregate is returned. + * + * \tparam LENGTH Length of input array + * \tparam T [inferred] The data type to be reduced. + * \tparam ScanOp [inferred] Binary reduction operator type having member T operator()(const T &a, const T &b) + */ +template < + int LENGTH, + typename T, + typename ReductionOp> +__device__ __forceinline__ T ThreadReduce( + T* input, ///< [in] Input array + ReductionOp reduction_op) ///< [in] Binary reduction operator +{ + T prefix = input[0]; + return ThreadReduce(input + 1, reduction_op, prefix); +} + + +/** + * \brief Perform a sequential reduction over the statically-sized \p input array, seeded with the specified \p prefix. The aggregate is returned. + * + * \tparam LENGTH [inferred] Length of \p input array + * \tparam T [inferred] The data type to be reduced. + * \tparam ScanOp [inferred] Binary reduction operator type having member T operator()(const T &a, const T &b) + */ +template < + int LENGTH, + typename T, + typename ReductionOp> +__device__ __forceinline__ T ThreadReduce( + T (&input)[LENGTH], ///< [in] Input array + ReductionOp reduction_op, ///< [in] Binary reduction operator + T prefix) ///< [in] Prefix to seed reduction with +{ + return ThreadReduce(input, reduction_op, prefix); +} + + +/** + * \brief Serial reduction with the specified operator + * + * \tparam LENGTH [inferred] Length of \p input array + * \tparam T [inferred] The data type to be reduced. + * \tparam ScanOp [inferred] Binary reduction operator type having member T operator()(const T &a, const T &b) + */ +template < + int LENGTH, + typename T, + typename ReductionOp> +__device__ __forceinline__ T ThreadReduce( + T (&input)[LENGTH], ///< [in] Input array + ReductionOp reduction_op) ///< [in] Binary reduction operator +{ + return ThreadReduce((T*) input, reduction_op); +} + + +//@} end member group + +/** @} */ // end group ThreadModule + +} // CUB namespace +CUB_NS_POSTFIX // Optional outer namespace(s) diff --git a/lib/kokkos/TPL/cub/thread/thread_scan.cuh b/lib/kokkos/TPL/cub/thread/thread_scan.cuh new file mode 100755 index 0000000000..b43bbcf00e --- /dev/null +++ b/lib/kokkos/TPL/cub/thread/thread_scan.cuh @@ -0,0 +1,231 @@ +/****************************************************************************** + * Copyright (c) 2011, Duane Merrill. All rights reserved. + * Copyright (c) 2011-2013, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the NVIDIA CORPORATION nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + ******************************************************************************/ + +/** + * \file + * Thread utilities for sequential prefix scan over statically-sized array types + */ + +#pragma once + +#include "../thread/thread_operators.cuh" +#include "../util_namespace.cuh" + +/// Optional outer namespace(s) +CUB_NS_PREFIX + +/// CUB namespace +namespace cub { + +/** + * \addtogroup ThreadModule + * @{ + */ + +/** + * \name Sequential prefix scan over statically-sized array types + * @{ + */ + +/** + * \brief Perform a sequential exclusive prefix scan over \p LENGTH elements of the \p input array, seeded with the specified \p prefix. The aggregate is returned. + * + * \tparam LENGTH Length of \p input and \p output arrays + * \tparam T [inferred] The data type to be scanned. + * \tparam ScanOp [inferred] Binary scan operator type having member T operator()(const T &a, const T &b) + */ +template < + int LENGTH, + typename T, + typename ScanOp> +__device__ __forceinline__ T ThreadScanExclusive( + T *input, ///< [in] Input array + T *output, ///< [out] Output array (may be aliased to \p input) + ScanOp scan_op, ///< [in] Binary scan operator + T prefix, ///< [in] Prefix to seed scan with + bool apply_prefix = true) ///< [in] Whether or not the calling thread should apply its prefix. If not, the first output element is undefined. (Handy for preventing thread-0 from applying a prefix.) +{ + T inclusive = input[0]; + if (apply_prefix) + { + inclusive = scan_op(prefix, inclusive); + } + output[0] = prefix; + T exclusive = inclusive; + + #pragma unroll + for (int i = 1; i < LENGTH; ++i) + { + inclusive = scan_op(exclusive, input[i]); + output[i] = exclusive; + exclusive = inclusive; + } + + return inclusive; +} + + +/** + * \brief Perform a sequential exclusive prefix scan over the statically-sized \p input array, seeded with the specified \p prefix. The aggregate is returned. + * + * \tparam LENGTH [inferred] Length of \p input and \p output arrays + * \tparam T [inferred] The data type to be scanned. + * \tparam ScanOp [inferred] Binary scan operator type having member T operator()(const T &a, const T &b) + */ +template < + int LENGTH, + typename T, + typename ScanOp> +__device__ __forceinline__ T ThreadScanExclusive( + T (&input)[LENGTH], ///< [in] Input array + T (&output)[LENGTH], ///< [out] Output array (may be aliased to \p input) + ScanOp scan_op, ///< [in] Binary scan operator + T prefix, ///< [in] Prefix to seed scan with + bool apply_prefix = true) ///< [in] Whether or not the calling thread should apply its prefix. (Handy for preventing thread-0 from applying a prefix.) +{ + return ThreadScanExclusive((T*) input, (T*) output, scan_op, prefix); +} + + +/** + * \brief Perform a sequential inclusive prefix scan over \p LENGTH elements of the \p input array. The aggregate is returned. + * + * \tparam LENGTH Length of \p input and \p output arrays + * \tparam T [inferred] The data type to be scanned. + * \tparam ScanOp [inferred] Binary scan operator type having member T operator()(const T &a, const T &b) + */ +template < + int LENGTH, + typename T, + typename ScanOp> +__device__ __forceinline__ T ThreadScanInclusive( + T *input, ///< [in] Input array + T *output, ///< [out] Output array (may be aliased to \p input) + ScanOp scan_op) ///< [in] Binary scan operator +{ + T inclusive = input[0]; + output[0] = inclusive; + + // Continue scan + #pragma unroll + for (int i = 0; i < LENGTH; ++i) + { + inclusive = scan_op(inclusive, input[i]); + output[i] = inclusive; + } + + return inclusive; +} + + +/** + * \brief Perform a sequential inclusive prefix scan over the statically-sized \p input array. The aggregate is returned. + * + * \tparam LENGTH [inferred] Length of \p input and \p output arrays + * \tparam T [inferred] The data type to be scanned. + * \tparam ScanOp [inferred] Binary scan operator type having member T operator()(const T &a, const T &b) + */ +template < + int LENGTH, + typename T, + typename ScanOp> +__device__ __forceinline__ T ThreadScanInclusive( + T (&input)[LENGTH], ///< [in] Input array + T (&output)[LENGTH], ///< [out] Output array (may be aliased to \p input) + ScanOp scan_op) ///< [in] Binary scan operator +{ + return ThreadScanInclusive((T*) input, (T*) output, scan_op); +} + + +/** + * \brief Perform a sequential inclusive prefix scan over \p LENGTH elements of the \p input array, seeded with the specified \p prefix. The aggregate is returned. + * + * \tparam LENGTH Length of \p input and \p output arrays + * \tparam T [inferred] The data type to be scanned. + * \tparam ScanOp [inferred] Binary scan operator type having member T operator()(const T &a, const T &b) + */ +template < + int LENGTH, + typename T, + typename ScanOp> +__device__ __forceinline__ T ThreadScanInclusive( + T *input, ///< [in] Input array + T *output, ///< [out] Output array (may be aliased to \p input) + ScanOp scan_op, ///< [in] Binary scan operator + T prefix, ///< [in] Prefix to seed scan with + bool apply_prefix = true) ///< [in] Whether or not the calling thread should apply its prefix. (Handy for preventing thread-0 from applying a prefix.) +{ + T inclusive = input[0]; + if (apply_prefix) + { + inclusive = scan_op(prefix, inclusive); + } + output[0] = inclusive; + + // Continue scan + #pragma unroll + for (int i = 1; i < LENGTH; ++i) + { + inclusive = scan_op(inclusive, input[i]); + output[i] = inclusive; + } + + return inclusive; +} + + +/** + * \brief Perform a sequential inclusive prefix scan over the statically-sized \p input array, seeded with the specified \p prefix. The aggregate is returned. + * + * \tparam LENGTH [inferred] Length of \p input and \p output arrays + * \tparam T [inferred] The data type to be scanned. + * \tparam ScanOp [inferred] Binary scan operator type having member T operator()(const T &a, const T &b) + */ +template < + int LENGTH, + typename T, + typename ScanOp> +__device__ __forceinline__ T ThreadScanInclusive( + T (&input)[LENGTH], ///< [in] Input array + T (&output)[LENGTH], ///< [out] Output array (may be aliased to \p input) + ScanOp scan_op, ///< [in] Binary scan operator + T prefix, ///< [in] Prefix to seed scan with + bool apply_prefix = true) ///< [in] Whether or not the calling thread should apply its prefix. (Handy for preventing thread-0 from applying a prefix.) +{ + return ThreadScanInclusive((T*) input, (T*) output, scan_op, prefix, apply_prefix); +} + + +//@} end member group + +/** @} */ // end group ThreadModule + + +} // CUB namespace +CUB_NS_POSTFIX // Optional outer namespace(s) diff --git a/lib/kokkos/TPL/cub/thread/thread_store.cuh b/lib/kokkos/TPL/cub/thread/thread_store.cuh new file mode 100755 index 0000000000..8d39e07b1d --- /dev/null +++ b/lib/kokkos/TPL/cub/thread/thread_store.cuh @@ -0,0 +1,412 @@ +/****************************************************************************** + * Copyright (c) 2011, Duane Merrill. All rights reserved. + * Copyright (c) 2011-2013, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the NVIDIA CORPORATION nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + ******************************************************************************/ + +/** + * \file + * Thread utilities for writing memory using PTX cache modifiers. + */ + +#pragma once + +#include + +#include "../util_ptx.cuh" +#include "../util_type.cuh" +#include "../util_namespace.cuh" + +/// Optional outer namespace(s) +CUB_NS_PREFIX + +/// CUB namespace +namespace cub { + +/** + * \addtogroup IoModule + * @{ + */ + + +//----------------------------------------------------------------------------- +// Tags and constants +//----------------------------------------------------------------------------- + +/** + * \brief Enumeration of PTX cache-modifiers for memory store operations. + */ +enum PtxStoreModifier +{ + STORE_DEFAULT, ///< Default (no modifier) + STORE_WB, ///< Cache write-back all coherent levels + STORE_CG, ///< Cache at global level + STORE_CS, ///< Cache streaming (likely to be accessed once) + STORE_WT, ///< Cache write-through (to system memory) + STORE_VOLATILE, ///< Volatile shared (any memory space) +}; + + +/** + * \name Simple I/O + * @{ + */ + +/** + * \brief Thread utility for writing memory using cub::PtxStoreModifier cache modifiers. + * + * Cache modifiers will only be effected for built-in types (i.e., C++ + * primitives and CUDA vector-types). + * + * For example: + * \par + * \code + * #include + * + * // 32-bit store using cache-global modifier: + * int *d_out; + * int val; + * cub::ThreadStore(d_out + threadIdx.x, val); + * + * // 16-bit store using default modifier + * short *d_out; + * short val; + * cub::ThreadStore(d_out + threadIdx.x, val); + * + * // 256-bit store using write-through modifier + * double4 *d_out; + * double4 val; + * cub::ThreadStore(d_out + threadIdx.x, val); + * + * // 96-bit store using default cache modifier (ignoring STORE_CS) + * struct TestFoo { bool a; short b; }; + * TestFoo *d_struct; + * TestFoo val; + * cub::ThreadStore(d_out + threadIdx.x, val); + * \endcode + * + */ +template < + PtxStoreModifier MODIFIER, + typename OutputIteratorRA, + typename T> +__device__ __forceinline__ void ThreadStore(OutputIteratorRA itr, T val); + + +//@} end member group + + +#ifndef DOXYGEN_SHOULD_SKIP_THIS // Do not document + + +/** + * Define a int4 (16B) ThreadStore specialization for the given PTX load modifier + */ +#define CUB_STORE_16(cub_modifier, ptx_modifier) \ + template<> \ + __device__ __forceinline__ void ThreadStore(int4* ptr, int4 val) \ + { \ + asm volatile ("st."#ptx_modifier".v4.s32 [%0], {%1, %2, %3, %4};" : : \ + _CUB_ASM_PTR_(ptr), \ + "r"(val.x), \ + "r"(val.y), \ + "r"(val.z), \ + "r"(val.w)); \ + } \ + template<> \ + __device__ __forceinline__ void ThreadStore(longlong2* ptr, longlong2 val) \ + { \ + asm volatile ("st."#ptx_modifier".v2.s64 [%0], {%1, %2};" : : \ + _CUB_ASM_PTR_(ptr), \ + "l"(val.x), \ + "l"(val.y)); \ + } + + +/** + * Define a int2 (8B) ThreadStore specialization for the given PTX load modifier + */ +#define CUB_STORE_8(cub_modifier, ptx_modifier) \ + template<> \ + __device__ __forceinline__ void ThreadStore(short4* ptr, short4 val) \ + { \ + asm volatile ("st."#ptx_modifier".v4.s16 [%0], {%1, %2, %3, %4};" : : \ + _CUB_ASM_PTR_(ptr), \ + "h"(val.x), \ + "h"(val.y), \ + "h"(val.z), \ + "h"(val.w)); \ + } \ + template<> \ + __device__ __forceinline__ void ThreadStore(int2* ptr, int2 val) \ + { \ + asm volatile ("st."#ptx_modifier".v2.s32 [%0], {%1, %2};" : : \ + _CUB_ASM_PTR_(ptr), \ + "r"(val.x), \ + "r"(val.y)); \ + } \ + template<> \ + __device__ __forceinline__ void ThreadStore(long long* ptr, long long val) \ + { \ + asm volatile ("st."#ptx_modifier".s64 [%0], %1;" : : \ + _CUB_ASM_PTR_(ptr), \ + "l"(val)); \ + } + +/** + * Define a int (4B) ThreadStore specialization for the given PTX load modifier + */ +#define CUB_STORE_4(cub_modifier, ptx_modifier) \ + template<> \ + __device__ __forceinline__ void ThreadStore(int* ptr, int val) \ + { \ + asm volatile ("st."#ptx_modifier".s32 [%0], %1;" : : \ + _CUB_ASM_PTR_(ptr), \ + "r"(val)); \ + } + + +/** + * Define a short (2B) ThreadStore specialization for the given PTX load modifier + */ +#define CUB_STORE_2(cub_modifier, ptx_modifier) \ + template<> \ + __device__ __forceinline__ void ThreadStore(short* ptr, short val) \ + { \ + asm volatile ("st."#ptx_modifier".s16 [%0], %1;" : : \ + _CUB_ASM_PTR_(ptr), \ + "h"(val)); \ + } + + +/** + * Define a char (1B) ThreadStore specialization for the given PTX load modifier + */ +#define CUB_STORE_1(cub_modifier, ptx_modifier) \ + template<> \ + __device__ __forceinline__ void ThreadStore(char* ptr, char val) \ + { \ + asm volatile ( \ + "{" \ + " .reg .s8 datum;" \ + " cvt.s8.s16 datum, %1;" \ + " st."#ptx_modifier".s8 [%0], datum;" \ + "}" : : \ + _CUB_ASM_PTR_(ptr), \ + "h"(short(val))); \ + } + +/** + * Define powers-of-two ThreadStore specializations for the given PTX load modifier + */ +#define CUB_STORE_ALL(cub_modifier, ptx_modifier) \ + CUB_STORE_16(cub_modifier, ptx_modifier) \ + CUB_STORE_8(cub_modifier, ptx_modifier) \ + CUB_STORE_4(cub_modifier, ptx_modifier) \ + CUB_STORE_2(cub_modifier, ptx_modifier) \ + CUB_STORE_1(cub_modifier, ptx_modifier) \ + + +/** + * Define ThreadStore specializations for the various PTX load modifiers + */ +#if CUB_PTX_ARCH >= 200 + CUB_STORE_ALL(STORE_WB, ca) + CUB_STORE_ALL(STORE_CG, cg) + CUB_STORE_ALL(STORE_CS, cs) + CUB_STORE_ALL(STORE_WT, cv) +#else + // STORE_WT on SM10-13 uses "volatile.global" to ensure writes to last level + CUB_STORE_ALL(STORE_WT, volatile.global) +#endif + + + +/// Helper structure for templated store iteration (inductive case) +template +struct IterateThreadStore +{ + template + static __device__ __forceinline__ void Store(T *ptr, T *vals) + { + ThreadStore(ptr + COUNT, vals[COUNT]); + IterateThreadStore::Store(ptr, vals); + } +}; + +/// Helper structure for templated store iteration (termination case) +template +struct IterateThreadStore +{ + template + static __device__ __forceinline__ void Store(T *ptr, T *vals) {} +}; + + + + +/** + * Store with STORE_DEFAULT on iterator types + */ +template +__device__ __forceinline__ void ThreadStore( + OutputIteratorRA itr, + T val, + Int2Type modifier, + Int2Type is_pointer) +{ + *itr = val; +} + + +/** + * Store with STORE_DEFAULT on pointer types + */ +template +__device__ __forceinline__ void ThreadStore( + T *ptr, + T val, + Int2Type modifier, + Int2Type is_pointer) +{ + *ptr = val; +} + + +/** + * Store with STORE_VOLATILE on primitive pointer types + */ +template +__device__ __forceinline__ void ThreadStoreVolatile( + T *ptr, + T val, + Int2Type is_primitive) +{ + *reinterpret_cast(ptr) = val; +} + + +/** + * Store with STORE_VOLATILE on non-primitive pointer types + */ +template +__device__ __forceinline__ void ThreadStoreVolatile( + T *ptr, + T val, + Int2Type is_primitive) +{ + typedef typename WordAlignment::VolatileWord VolatileWord; // Word type for memcopying + enum { NUM_WORDS = sizeof(T) / sizeof(VolatileWord) }; + + // Store into array of uninitialized words + typename WordAlignment::UninitializedVolatileWords words; + *reinterpret_cast(words.buf) = val; + + // Memcopy words to aliased destination + #pragma unroll + for (int i = 0; i < NUM_WORDS; ++i) + reinterpret_cast(ptr)[i] = words.buf[i]; +} + + +/** + * Store with STORE_VOLATILE on pointer types + */ +template +__device__ __forceinline__ void ThreadStore( + T *ptr, + T val, + Int2Type modifier, + Int2Type is_pointer) +{ + ThreadStoreVolatile(ptr, val, Int2Type::PRIMITIVE>()); +} + + +#if (CUB_PTX_ARCH <= 350) + +/** + * Store with STORE_CG on pointer types (uses STORE_DEFAULT on current architectures) + */ +template +__device__ __forceinline__ void ThreadStore( + T *ptr, + T val, + Int2Type modifier, + Int2Type is_pointer) +{ + ThreadStore(ptr, val); +} + +#endif // (CUB_PTX_ARCH <= 350) + + +/** + * Store with arbitrary MODIFIER on pointer types + */ +template +__device__ __forceinline__ void ThreadStore( + T *ptr, + T val, + Int2Type modifier, + Int2Type is_pointer) +{ + typedef typename WordAlignment::DeviceWord DeviceWord; // Word type for memcopying + enum { NUM_WORDS = sizeof(T) / sizeof(DeviceWord) }; + + // Store into array of uninitialized words + typename WordAlignment::UninitializedDeviceWords words; + *reinterpret_cast(words.buf) = val; + + // Memcopy words to aliased destination + IterateThreadStore::Store( + reinterpret_cast(ptr), + words.buf); +} + + +/** + * Generic ThreadStore definition + */ +template +__device__ __forceinline__ void ThreadStore(OutputIteratorRA itr, T val) +{ + ThreadStore( + itr, + val, + Int2Type(), + Int2Type::VALUE>()); +} + + + +#endif // DOXYGEN_SHOULD_SKIP_THIS + + +/** @} */ // end group IoModule + + +} // CUB namespace +CUB_NS_POSTFIX // Optional outer namespace(s) diff --git a/lib/kokkos/TPL/cub/util_allocator.cuh b/lib/kokkos/TPL/cub/util_allocator.cuh new file mode 100755 index 0000000000..ae40f33050 --- /dev/null +++ b/lib/kokkos/TPL/cub/util_allocator.cuh @@ -0,0 +1,661 @@ +/****************************************************************************** + * Copyright (c) 2011, Duane Merrill. All rights reserved. + * Copyright (c) 2011-2013, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the NVIDIA CORPORATION nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + ******************************************************************************/ + +/****************************************************************************** + * Simple caching allocator for device memory allocations. The allocator is + * thread-safe and capable of managing device allocations on multiple devices. + ******************************************************************************/ + +#pragma once + +#ifndef __CUDA_ARCH__ + #include // NVCC (EDG, really) takes FOREVER to compile std::map + #include +#endif + +#include + +#include "util_namespace.cuh" +#include "util_debug.cuh" + +#include "host/spinlock.cuh" + +/// Optional outer namespace(s) +CUB_NS_PREFIX + +/// CUB namespace +namespace cub { + + +/** + * \addtogroup UtilModule + * @{ + */ + + +/****************************************************************************** + * CachingDeviceAllocator (host use) + ******************************************************************************/ + +/** + * \brief A simple caching allocator for device memory allocations. + * + * \par Overview + * The allocator is thread-safe and is capable of managing cached device allocations + * on multiple devices. It behaves as follows: + * + * \par + * - Allocations categorized by bin size. + * - Bin sizes progress geometrically in accordance with the growth factor + * \p bin_growth provided during construction. Unused device allocations within + * a larger bin cache are not reused for allocation requests that categorize to + * smaller bin sizes. + * - Allocation requests below (\p bin_growth ^ \p min_bin) are rounded up to + * (\p bin_growth ^ \p min_bin). + * - Allocations above (\p bin_growth ^ \p max_bin) are not rounded up to the nearest + * bin and are simply freed when they are deallocated instead of being returned + * to a bin-cache. + * - %If the total storage of cached allocations on a given device will exceed + * \p max_cached_bytes, allocations for that device are simply freed when they are + * deallocated instead of being returned to their bin-cache. + * + * \par + * For example, the default-constructed CachingDeviceAllocator is configured with: + * - \p bin_growth = 8 + * - \p min_bin = 3 + * - \p max_bin = 7 + * - \p max_cached_bytes = 6MB - 1B + * + * \par + * which delineates five bin-sizes: 512B, 4KB, 32KB, 256KB, and 2MB + * and sets a maximum of 6,291,455 cached bytes per device + * + */ +struct CachingDeviceAllocator +{ +#ifndef DOXYGEN_SHOULD_SKIP_THIS // Do not document + + + //--------------------------------------------------------------------- + // Type definitions and constants + //--------------------------------------------------------------------- + + enum + { + /// Invalid device ordinal + INVALID_DEVICE_ORDINAL = -1, + }; + + /** + * Integer pow function for unsigned base and exponent + */ + static unsigned int IntPow( + unsigned int base, + unsigned int exp) + { + unsigned int retval = 1; + while (exp > 0) + { + if (exp & 1) { + retval = retval * base; // multiply the result by the current base + } + base = base * base; // square the base + exp = exp >> 1; // divide the exponent in half + } + return retval; + } + + + /** + * Round up to the nearest power-of + */ + static void NearestPowerOf( + unsigned int &power, + size_t &rounded_bytes, + unsigned int base, + size_t value) + { + power = 0; + rounded_bytes = 1; + + while (rounded_bytes < value) + { + rounded_bytes *= base; + power++; + } + } + + /** + * Descriptor for device memory allocations + */ + struct BlockDescriptor + { + int device; // device ordinal + void* d_ptr; // Device pointer + size_t bytes; // Size of allocation in bytes + unsigned int bin; // Bin enumeration + + // Constructor + BlockDescriptor(void *d_ptr, int device) : + d_ptr(d_ptr), + bytes(0), + bin(0), + device(device) {} + + // Constructor + BlockDescriptor(size_t bytes, unsigned int bin, int device) : + d_ptr(NULL), + bytes(bytes), + bin(bin), + device(device) {} + + // Comparison functor for comparing device pointers + static bool PtrCompare(const BlockDescriptor &a, const BlockDescriptor &b) + { + if (a.device < b.device) { + return true; + } else if (a.device > b.device) { + return false; + } else { + return (a.d_ptr < b.d_ptr); + } + } + + // Comparison functor for comparing allocation sizes + static bool SizeCompare(const BlockDescriptor &a, const BlockDescriptor &b) + { + if (a.device < b.device) { + return true; + } else if (a.device > b.device) { + return false; + } else { + return (a.bytes < b.bytes); + } + } + }; + + /// BlockDescriptor comparator function interface + typedef bool (*Compare)(const BlockDescriptor &, const BlockDescriptor &); + +#ifndef __CUDA_ARCH__ // Only define STL container members in host code + + /// Set type for cached blocks (ordered by size) + typedef std::multiset CachedBlocks; + + /// Set type for live blocks (ordered by ptr) + typedef std::multiset BusyBlocks; + + /// Map type of device ordinals to the number of cached bytes cached by each device + typedef std::map GpuCachedBytes; + +#endif // __CUDA_ARCH__ + + //--------------------------------------------------------------------- + // Fields + //--------------------------------------------------------------------- + + Spinlock spin_lock; /// Spinlock for thread-safety + + unsigned int bin_growth; /// Geometric growth factor for bin-sizes + unsigned int min_bin; /// Minimum bin enumeration + unsigned int max_bin; /// Maximum bin enumeration + + size_t min_bin_bytes; /// Minimum bin size + size_t max_bin_bytes; /// Maximum bin size + size_t max_cached_bytes; /// Maximum aggregate cached bytes per device + + bool debug; /// Whether or not to print (de)allocation events to stdout + bool skip_cleanup; /// Whether or not to skip a call to FreeAllCached() when destructor is called. (The CUDA runtime may have already shut down for statically declared allocators) + +#ifndef __CUDA_ARCH__ // Only define STL container members in host code + + GpuCachedBytes cached_bytes; /// Map of device ordinal to aggregate cached bytes on that device + CachedBlocks cached_blocks; /// Set of cached device allocations available for reuse + BusyBlocks live_blocks; /// Set of live device allocations currently in use + +#endif // __CUDA_ARCH__ + +#endif // DOXYGEN_SHOULD_SKIP_THIS + + //--------------------------------------------------------------------- + // Methods + //--------------------------------------------------------------------- + + /** + * \brief Constructor. + */ + CachingDeviceAllocator( + unsigned int bin_growth, ///< Geometric growth factor for bin-sizes + unsigned int min_bin, ///< Minimum bin + unsigned int max_bin, ///< Maximum bin + size_t max_cached_bytes) ///< Maximum aggregate cached bytes per device + : + #ifndef __CUDA_ARCH__ // Only define STL container members in host code + cached_blocks(BlockDescriptor::SizeCompare), + live_blocks(BlockDescriptor::PtrCompare), + #endif + debug(false), + spin_lock(0), + bin_growth(bin_growth), + min_bin(min_bin), + max_bin(max_bin), + min_bin_bytes(IntPow(bin_growth, min_bin)), + max_bin_bytes(IntPow(bin_growth, max_bin)), + max_cached_bytes(max_cached_bytes) + {} + + + /** + * \brief Default constructor. + * + * Configured with: + * \par + * - \p bin_growth = 8 + * - \p min_bin = 3 + * - \p max_bin = 7 + * - \p max_cached_bytes = (\p bin_growth ^ \p max_bin) * 3) - 1 = 6,291,455 bytes + * + * which delineates five bin-sizes: 512B, 4KB, 32KB, 256KB, and 2MB and + * sets a maximum of 6,291,455 cached bytes per device + */ + CachingDeviceAllocator(bool skip_cleanup = false) : + #ifndef __CUDA_ARCH__ // Only define STL container members in host code + cached_blocks(BlockDescriptor::SizeCompare), + live_blocks(BlockDescriptor::PtrCompare), + #endif + skip_cleanup(skip_cleanup), + debug(false), + spin_lock(0), + bin_growth(8), + min_bin(3), + max_bin(7), + min_bin_bytes(IntPow(bin_growth, min_bin)), + max_bin_bytes(IntPow(bin_growth, max_bin)), + max_cached_bytes((max_bin_bytes * 3) - 1) + {} + + + /** + * \brief Sets the limit on the number bytes this allocator is allowed to cache per device. + */ + cudaError_t SetMaxCachedBytes( + size_t max_cached_bytes) + { + #ifdef __CUDA_ARCH__ + // Caching functionality only defined on host + return CubDebug(cudaErrorInvalidConfiguration); + #else + + // Lock + Lock(&spin_lock); + + this->max_cached_bytes = max_cached_bytes; + + if (debug) CubLog("New max_cached_bytes(%lld)\n", (long long) max_cached_bytes); + + // Unlock + Unlock(&spin_lock); + + return cudaSuccess; + + #endif // __CUDA_ARCH__ + } + + + /** + * \brief Provides a suitable allocation of device memory for the given size on the specified device + */ + cudaError_t DeviceAllocate( + void** d_ptr, + size_t bytes, + int device) + { + #ifdef __CUDA_ARCH__ + // Caching functionality only defined on host + return CubDebug(cudaErrorInvalidConfiguration); + #else + + bool locked = false; + int entrypoint_device = INVALID_DEVICE_ORDINAL; + cudaError_t error = cudaSuccess; + + // Round up to nearest bin size + unsigned int bin; + size_t bin_bytes; + NearestPowerOf(bin, bin_bytes, bin_growth, bytes); + if (bin < min_bin) { + bin = min_bin; + bin_bytes = min_bin_bytes; + } + + // Check if bin is greater than our maximum bin + if (bin > max_bin) + { + // Allocate the request exactly and give out-of-range bin + bin = (unsigned int) -1; + bin_bytes = bytes; + } + + BlockDescriptor search_key(bin_bytes, bin, device); + + // Lock + if (!locked) { + Lock(&spin_lock); + locked = true; + } + + do { + // Find a free block big enough within the same bin on the same device + CachedBlocks::iterator block_itr = cached_blocks.lower_bound(search_key); + if ((block_itr != cached_blocks.end()) && + (block_itr->device == device) && + (block_itr->bin == search_key.bin)) + { + // Reuse existing cache block. Insert into live blocks. + search_key = *block_itr; + live_blocks.insert(search_key); + + // Remove from free blocks + cached_blocks.erase(block_itr); + cached_bytes[device] -= search_key.bytes; + + if (debug) CubLog("\tdevice %d reused cached block (%lld bytes). %lld available blocks cached (%lld bytes), %lld live blocks outstanding.\n", + device, (long long) search_key.bytes, (long long) cached_blocks.size(), (long long) cached_bytes[device], (long long) live_blocks.size()); + } + else + { + // Need to allocate a new cache block. Unlock. + if (locked) { + Unlock(&spin_lock); + locked = false; + } + + // Set to specified device + if (CubDebug(error = cudaGetDevice(&entrypoint_device))) break; + if (CubDebug(error = cudaSetDevice(device))) break; + + // Allocate + if (CubDebug(error = cudaMalloc(&search_key.d_ptr, search_key.bytes))) break; + + // Lock + if (!locked) { + Lock(&spin_lock); + locked = true; + } + + // Insert into live blocks + live_blocks.insert(search_key); + + if (debug) CubLog("\tdevice %d allocating new device block %lld bytes. %lld available blocks cached (%lld bytes), %lld live blocks outstanding.\n", + device, (long long) search_key.bytes, (long long) cached_blocks.size(), (long long) cached_bytes[device], (long long) live_blocks.size()); + } + } while(0); + + // Unlock + if (locked) { + Unlock(&spin_lock); + locked = false; + } + + // Copy device pointer to output parameter (NULL on error) + *d_ptr = search_key.d_ptr; + + // Attempt to revert back to previous device if necessary + if (entrypoint_device != INVALID_DEVICE_ORDINAL) + { + if (CubDebug(error = cudaSetDevice(entrypoint_device))) return error; + } + + return error; + + #endif // __CUDA_ARCH__ + } + + + /** + * \brief Provides a suitable allocation of device memory for the given size on the current device + */ + cudaError_t DeviceAllocate( + void** d_ptr, + size_t bytes) + { + #ifdef __CUDA_ARCH__ + // Caching functionality only defined on host + return CubDebug(cudaErrorInvalidConfiguration); + #else + cudaError_t error = cudaSuccess; + do { + int current_device; + if (CubDebug(error = cudaGetDevice(¤t_device))) break; + if (CubDebug(error = DeviceAllocate(d_ptr, bytes, current_device))) break; + } while(0); + + return error; + + #endif // __CUDA_ARCH__ + } + + + /** + * \brief Frees a live allocation of device memory on the specified device, returning it to the allocator + */ + cudaError_t DeviceFree( + void* d_ptr, + int device) + { + #ifdef __CUDA_ARCH__ + // Caching functionality only defined on host + return CubDebug(cudaErrorInvalidConfiguration); + #else + + bool locked = false; + int entrypoint_device = INVALID_DEVICE_ORDINAL; + cudaError_t error = cudaSuccess; + + BlockDescriptor search_key(d_ptr, device); + + // Lock + if (!locked) { + Lock(&spin_lock); + locked = true; + } + + do { + // Find corresponding block descriptor + BusyBlocks::iterator block_itr = live_blocks.find(search_key); + if (block_itr == live_blocks.end()) + { + // Cannot find pointer + if (CubDebug(error = cudaErrorUnknown)) break; + } + else + { + // Remove from live blocks + search_key = *block_itr; + live_blocks.erase(block_itr); + + // Check if we should keep the returned allocation + if (cached_bytes[device] + search_key.bytes <= max_cached_bytes) + { + // Insert returned allocation into free blocks + cached_blocks.insert(search_key); + cached_bytes[device] += search_key.bytes; + + if (debug) CubLog("\tdevice %d returned %lld bytes. %lld available blocks cached (%lld bytes), %lld live blocks outstanding.\n", + device, (long long) search_key.bytes, (long long) cached_blocks.size(), (long long) cached_bytes[device], (long long) live_blocks.size()); + } + else + { + // Free the returned allocation. Unlock. + if (locked) { + Unlock(&spin_lock); + locked = false; + } + + // Set to specified device + if (CubDebug(error = cudaGetDevice(&entrypoint_device))) break; + if (CubDebug(error = cudaSetDevice(device))) break; + + // Free device memory + if (CubDebug(error = cudaFree(d_ptr))) break; + + if (debug) CubLog("\tdevice %d freed %lld bytes. %lld available blocks cached (%lld bytes), %lld live blocks outstanding.\n", + device, (long long) search_key.bytes, (long long) cached_blocks.size(), (long long) cached_bytes[device], (long long) live_blocks.size()); + } + } + } while (0); + + // Unlock + if (locked) { + Unlock(&spin_lock); + locked = false; + } + + // Attempt to revert back to entry-point device if necessary + if (entrypoint_device != INVALID_DEVICE_ORDINAL) + { + if (CubDebug(error = cudaSetDevice(entrypoint_device))) return error; + } + + return error; + + #endif // __CUDA_ARCH__ + } + + + /** + * \brief Frees a live allocation of device memory on the current device, returning it to the allocator + */ + cudaError_t DeviceFree( + void* d_ptr) + { + #ifdef __CUDA_ARCH__ + // Caching functionality only defined on host + return CubDebug(cudaErrorInvalidConfiguration); + #else + + int current_device; + cudaError_t error = cudaSuccess; + + do { + if (CubDebug(error = cudaGetDevice(¤t_device))) break; + if (CubDebug(error = DeviceFree(d_ptr, current_device))) break; + } while(0); + + return error; + + #endif // __CUDA_ARCH__ + } + + + /** + * \brief Frees all cached device allocations on all devices + */ + cudaError_t FreeAllCached() + { + #ifdef __CUDA_ARCH__ + // Caching functionality only defined on host + return CubDebug(cudaErrorInvalidConfiguration); + #else + + cudaError_t error = cudaSuccess; + bool locked = false; + int entrypoint_device = INVALID_DEVICE_ORDINAL; + int current_device = INVALID_DEVICE_ORDINAL; + + // Lock + if (!locked) { + Lock(&spin_lock); + locked = true; + } + + while (!cached_blocks.empty()) + { + // Get first block + CachedBlocks::iterator begin = cached_blocks.begin(); + + // Get entry-point device ordinal if necessary + if (entrypoint_device == INVALID_DEVICE_ORDINAL) + { + if (CubDebug(error = cudaGetDevice(&entrypoint_device))) break; + } + + // Set current device ordinal if necessary + if (begin->device != current_device) + { + if (CubDebug(error = cudaSetDevice(begin->device))) break; + current_device = begin->device; + } + + // Free device memory + if (CubDebug(error = cudaFree(begin->d_ptr))) break; + + // Reduce balance and erase entry + cached_bytes[current_device] -= begin->bytes; + cached_blocks.erase(begin); + + if (debug) CubLog("\tdevice %d freed %lld bytes. %lld available blocks cached (%lld bytes), %lld live blocks outstanding.\n", + current_device, (long long) begin->bytes, (long long) cached_blocks.size(), (long long) cached_bytes[current_device], (long long) live_blocks.size()); + } + + // Unlock + if (locked) { + Unlock(&spin_lock); + locked = false; + } + + // Attempt to revert back to entry-point device if necessary + if (entrypoint_device != INVALID_DEVICE_ORDINAL) + { + if (CubDebug(error = cudaSetDevice(entrypoint_device))) return error; + } + + return error; + + #endif // __CUDA_ARCH__ + } + + + /** + * \brief Destructor + */ + virtual ~CachingDeviceAllocator() + { + if (!skip_cleanup) + FreeAllCached(); + } + +}; + + + + +/** @} */ // end group UtilModule + +} // CUB namespace +CUB_NS_POSTFIX // Optional outer namespace(s) diff --git a/lib/kokkos/TPL/cub/util_arch.cuh b/lib/kokkos/TPL/cub/util_arch.cuh new file mode 100755 index 0000000000..232a33c4f4 --- /dev/null +++ b/lib/kokkos/TPL/cub/util_arch.cuh @@ -0,0 +1,295 @@ +/****************************************************************************** + * Copyright (c) 2011, Duane Merrill. All rights reserved. + * Copyright (c) 2011-2013, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the NVIDIA CORPORATION nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + ******************************************************************************/ + +/** + * \file + * Static architectural properties by SM version. + */ + + +/****************************************************************************** + * Static architectural properties by SM version. + * + * "Device" reflects the PTX architecture targeted by the active compiler + * pass. It provides useful compile-time statics within device code. E.g.,: + * + * __shared__ int[Device::WARP_THREADS]; + * + * int padded_offset = threadIdx.x + (threadIdx.x >> Device::LOG_SMEM_BANKS); + * + ******************************************************************************/ + +#pragma once + +#include "util_namespace.cuh" + +/// Optional outer namespace(s) +CUB_NS_PREFIX + +/// CUB namespace +namespace cub { + + +/** + * \addtogroup UtilModule + * @{ + */ + + +/// CUB_PTX_ARCH reflects the PTX version targeted by the active compiler pass (or zero during the host pass). +#ifndef __CUDA_ARCH__ + #define CUB_PTX_ARCH 0 +#else + #define CUB_PTX_ARCH __CUDA_ARCH__ +#endif + + +/// Whether or not the source targeted by the active compiler pass is allowed to invoke device kernels or methods from the CUDA runtime API. +#if !defined(__CUDA_ARCH__) || defined(CUB_CDP) +#define CUB_RUNTIME_ENABLED +#endif + + +/// Execution space for destructors +#if ((CUB_PTX_ARCH > 0) && (CUB_PTX_ARCH < 200)) + #define CUB_DESTRUCTOR __host__ +#else + #define CUB_DESTRUCTOR __host__ __device__ +#endif + + +/** + * \brief Structure for statically reporting CUDA device properties, parameterized by SM architecture. + * + * The default specialization is for SM10. + */ +template +struct ArchProps +{ + enum + { + LOG_WARP_THREADS = + 5, /// Log of the number of threads per warp + WARP_THREADS = + 1 << LOG_WARP_THREADS, /// Number of threads per warp + LOG_SMEM_BANKS = + 4, /// Log of the number of smem banks + SMEM_BANKS = + 1 << LOG_SMEM_BANKS, /// The number of smem banks + SMEM_BANK_BYTES = + 4, /// Size of smem bank words + SMEM_BYTES = + 16 * 1024, /// Maximum SM shared memory + SMEM_ALLOC_UNIT = + 512, /// Smem allocation size in bytes + REGS_BY_BLOCK = + true, /// Whether or not the architecture allocates registers by block (or by warp) + REG_ALLOC_UNIT = + 256, /// Number of registers allocated at a time per block (or by warp) + WARP_ALLOC_UNIT = + 2, /// Granularity of warps for which registers are allocated + MAX_SM_THREADS = + 768, /// Maximum number of threads per SM + MAX_SM_THREADBLOCKS = + 8, /// Maximum number of thread blocks per SM + MAX_BLOCK_THREADS = + 512, /// Maximum number of thread per thread block + MAX_SM_REGISTERS = + 8 * 1024, /// Maximum number of registers per SM + }; +}; + + + + +#ifndef DOXYGEN_SHOULD_SKIP_THIS // Do not document + +/** + * Architecture properties for SM30 + */ +template <> +struct ArchProps<300> +{ + enum + { + LOG_WARP_THREADS = 5, // 32 threads per warp + WARP_THREADS = 1 << LOG_WARP_THREADS, + LOG_SMEM_BANKS = 5, // 32 banks + SMEM_BANKS = 1 << LOG_SMEM_BANKS, + SMEM_BANK_BYTES = 4, // 4 byte bank words + SMEM_BYTES = 48 * 1024, // 48KB shared memory + SMEM_ALLOC_UNIT = 256, // 256B smem allocation segment size + REGS_BY_BLOCK = false, // Allocates registers by warp + REG_ALLOC_UNIT = 256, // 256 registers allocated at a time per warp + WARP_ALLOC_UNIT = 4, // Registers are allocated at a granularity of every 4 warps per threadblock + MAX_SM_THREADS = 2048, // 2K max threads per SM + MAX_SM_THREADBLOCKS = 16, // 16 max threadblocks per SM + MAX_BLOCK_THREADS = 1024, // 1024 max threads per threadblock + MAX_SM_REGISTERS = 64 * 1024, // 64K max registers per SM + }; + + // Callback utility + template + static __host__ __device__ __forceinline__ void Callback(T &target, int sm_version) + { + target.template Callback(); + } +}; + + +/** + * Architecture properties for SM20 + */ +template <> +struct ArchProps<200> +{ + enum + { + LOG_WARP_THREADS = 5, // 32 threads per warp + WARP_THREADS = 1 << LOG_WARP_THREADS, + LOG_SMEM_BANKS = 5, // 32 banks + SMEM_BANKS = 1 << LOG_SMEM_BANKS, + SMEM_BANK_BYTES = 4, // 4 byte bank words + SMEM_BYTES = 48 * 1024, // 48KB shared memory + SMEM_ALLOC_UNIT = 128, // 128B smem allocation segment size + REGS_BY_BLOCK = false, // Allocates registers by warp + REG_ALLOC_UNIT = 64, // 64 registers allocated at a time per warp + WARP_ALLOC_UNIT = 2, // Registers are allocated at a granularity of every 2 warps per threadblock + MAX_SM_THREADS = 1536, // 1536 max threads per SM + MAX_SM_THREADBLOCKS = 8, // 8 max threadblocks per SM + MAX_BLOCK_THREADS = 1024, // 1024 max threads per threadblock + MAX_SM_REGISTERS = 32 * 1024, // 32K max registers per SM + }; + + // Callback utility + template + static __host__ __device__ __forceinline__ void Callback(T &target, int sm_version) + { + if (sm_version > 200) { + ArchProps<300>::Callback(target, sm_version); + } else { + target.template Callback(); + } + } +}; + + +/** + * Architecture properties for SM12 + */ +template <> +struct ArchProps<120> +{ + enum + { + LOG_WARP_THREADS = 5, // 32 threads per warp + WARP_THREADS = 1 << LOG_WARP_THREADS, + LOG_SMEM_BANKS = 4, // 16 banks + SMEM_BANKS = 1 << LOG_SMEM_BANKS, + SMEM_BANK_BYTES = 4, // 4 byte bank words + SMEM_BYTES = 16 * 1024, // 16KB shared memory + SMEM_ALLOC_UNIT = 512, // 512B smem allocation segment size + REGS_BY_BLOCK = true, // Allocates registers by threadblock + REG_ALLOC_UNIT = 512, // 512 registers allocated at time per threadblock + WARP_ALLOC_UNIT = 2, // Registers are allocated at a granularity of every 2 warps per threadblock + MAX_SM_THREADS = 1024, // 1024 max threads per SM + MAX_SM_THREADBLOCKS = 8, // 8 max threadblocks per SM + MAX_BLOCK_THREADS = 512, // 512 max threads per threadblock + MAX_SM_REGISTERS = 16 * 1024, // 16K max registers per SM + }; + + // Callback utility + template + static __host__ __device__ __forceinline__ void Callback(T &target, int sm_version) + { + if (sm_version > 120) { + ArchProps<200>::Callback(target, sm_version); + } else { + target.template Callback(); + } + } +}; + + +/** + * Architecture properties for SM10. Derives from the default ArchProps specialization. + */ +template <> +struct ArchProps<100> : ArchProps<0> +{ + // Callback utility + template + static __host__ __device__ __forceinline__ void Callback(T &target, int sm_version) + { + if (sm_version > 100) { + ArchProps<120>::Callback(target, sm_version); + } else { + target.template Callback(); + } + } +}; + + +/** + * Architecture properties for SM35 + */ +template <> +struct ArchProps<350> : ArchProps<300> {}; // Derives from SM30 + +/** + * Architecture properties for SM21 + */ +template <> +struct ArchProps<210> : ArchProps<200> {}; // Derives from SM20 + +/** + * Architecture properties for SM13 + */ +template <> +struct ArchProps<130> : ArchProps<120> {}; // Derives from SM12 + +/** + * Architecture properties for SM11 + */ +template <> +struct ArchProps<110> : ArchProps<100> {}; // Derives from SM10 + + +#endif // DOXYGEN_SHOULD_SKIP_THIS + + +/** + * \brief The architectural properties for the PTX version targeted by the active compiler pass. + */ +struct PtxArchProps : ArchProps {}; + + +/** @} */ // end group UtilModule + +} // CUB namespace +CUB_NS_POSTFIX // Optional outer namespace(s) diff --git a/lib/kokkos/TPL/cub/util_debug.cuh b/lib/kokkos/TPL/cub/util_debug.cuh new file mode 100755 index 0000000000..2ac67d7d04 --- /dev/null +++ b/lib/kokkos/TPL/cub/util_debug.cuh @@ -0,0 +1,115 @@ +/****************************************************************************** + * Copyright (c) 2011, Duane Merrill. All rights reserved. + * Copyright (c) 2011-2013, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the NVIDIA CORPORATION nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + ******************************************************************************/ + +/** + * \file + * Error and event logging routines. + * + * The following macros definitions are supported: + * - \p CUB_LOG. Simple event messages are printed to \p stdout. + */ + +#pragma once + +#include +#include "util_namespace.cuh" +#include "util_arch.cuh" + +/// Optional outer namespace(s) +CUB_NS_PREFIX + +/// CUB namespace +namespace cub { + + +/** + * \addtogroup UtilModule + * @{ + */ + + +/// CUB error reporting macro (prints error messages to stderr) +#if (defined(DEBUG) || defined(_DEBUG)) + #define CUB_STDERR +#endif + + + +/** + * \brief %If \p CUB_STDERR is defined and \p error is not \p cudaSuccess, the corresponding error message is printed to \p stderr (or \p stdout in device code) along with the supplied source context. + * + * \return The CUDA error. + */ +__host__ __device__ __forceinline__ cudaError_t Debug( + cudaError_t error, + const char* filename, + int line) +{ +#ifdef CUB_STDERR + if (error) + { + #if (CUB_PTX_ARCH == 0) + fprintf(stderr, "CUDA error %d [%s, %d]: %s\n", error, filename, line, cudaGetErrorString(error)); + fflush(stderr); + #elif (CUB_PTX_ARCH >= 200) + printf("CUDA error %d [block %d, thread %d, %s, %d]\n", error, blockIdx.x, threadIdx.x, filename, line); + #endif + } +#endif + return error; +} + + +/** + * \brief Debug macro + */ +#define CubDebug(e) cub::Debug((e), __FILE__, __LINE__) + + +/** + * \brief Debug macro with exit + */ +#define CubDebugExit(e) if (cub::Debug((e), __FILE__, __LINE__)) { exit(1); } + + +/** + * \brief Log macro for printf statements. + */ +#if (CUB_PTX_ARCH == 0) + #define CubLog(format, ...) printf(format,__VA_ARGS__); +#elif (CUB_PTX_ARCH >= 200) + #define CubLog(format, ...) printf("[block %d, thread %d]: " format, blockIdx.x, threadIdx.x, __VA_ARGS__); +#endif + + + + +/** @} */ // end group UtilModule + +} // CUB namespace +CUB_NS_POSTFIX // Optional outer namespace(s) diff --git a/lib/kokkos/TPL/cub/util_device.cuh b/lib/kokkos/TPL/cub/util_device.cuh new file mode 100755 index 0000000000..0631b924ae --- /dev/null +++ b/lib/kokkos/TPL/cub/util_device.cuh @@ -0,0 +1,378 @@ +/****************************************************************************** + * Copyright (c) 2011, Duane Merrill. All rights reserved. + * Copyright (c) 2011-2013, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the NVIDIA CORPORATION nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + ******************************************************************************/ + +/** + * \file + * Properties of a given CUDA device and the corresponding PTX bundle + */ + +#pragma once + +#include "util_arch.cuh" +#include "util_debug.cuh" +#include "util_namespace.cuh" +#include "util_macro.cuh" + +/// Optional outer namespace(s) +CUB_NS_PREFIX + +/// CUB namespace +namespace cub { + + +/** + * \addtogroup UtilModule + * @{ + */ + +#ifndef DOXYGEN_SHOULD_SKIP_THIS // Do not document + + +/** + * Empty kernel for querying PTX manifest metadata (e.g., version) for the current device + */ +template +__global__ void EmptyKernel(void) { } + + +/** + * Alias temporaries to externally-allocated device storage (or simply return the amount of storage needed). + */ +template +__host__ __device__ __forceinline__ +cudaError_t AliasTemporaries( + void *d_temp_storage, ///< [in] %Device allocation of temporary storage. When NULL, the required allocation size is returned in \p temp_storage_bytes and no work is done. + size_t &temp_storage_bytes, ///< [in,out] Size in bytes of \t d_temp_storage allocation + void* (&allocations)[ALLOCATIONS], ///< [in,out] Pointers to device allocations needed + size_t (&allocation_sizes)[ALLOCATIONS]) ///< [in] Sizes in bytes of device allocations needed +{ + const int ALIGN_BYTES = 256; + const int ALIGN_MASK = ~(ALIGN_BYTES - 1); + + // Compute exclusive prefix sum over allocation requests + size_t bytes_needed = 0; + for (int i = 0; i < ALLOCATIONS; ++i) + { + size_t allocation_bytes = (allocation_sizes[i] + ALIGN_BYTES - 1) & ALIGN_MASK; + allocation_sizes[i] = bytes_needed; + bytes_needed += allocation_bytes; + } + + // Check if the caller is simply requesting the size of the storage allocation + if (!d_temp_storage) + { + temp_storage_bytes = bytes_needed; + return cudaSuccess; + } + + // Check if enough storage provided + if (temp_storage_bytes < bytes_needed) + { + return CubDebug(cudaErrorMemoryAllocation); + } + + // Alias + for (int i = 0; i < ALLOCATIONS; ++i) + { + allocations[i] = static_cast(d_temp_storage) + allocation_sizes[i]; + } + + return cudaSuccess; +} + + + +#endif // DOXYGEN_SHOULD_SKIP_THIS + + + +/** + * \brief Retrieves the PTX version (major * 100 + minor * 10) + */ +__host__ __device__ __forceinline__ cudaError_t PtxVersion(int &ptx_version) +{ +#ifndef CUB_RUNTIME_ENABLED + + // CUDA API calls not supported from this device + return cudaErrorInvalidConfiguration; + +#else + + cudaError_t error = cudaSuccess; + do + { + cudaFuncAttributes empty_kernel_attrs; + if (CubDebug(error = cudaFuncGetAttributes(&empty_kernel_attrs, EmptyKernel))) break; + ptx_version = empty_kernel_attrs.ptxVersion * 10; + } + while (0); + + return error; + +#endif +} + + +/** + * Synchronize the stream if specified + */ +__host__ __device__ __forceinline__ +static cudaError_t SyncStream(cudaStream_t stream) +{ +#ifndef __CUDA_ARCH__ + return cudaStreamSynchronize(stream); +#else + // Device can't yet sync on a specific stream + return cudaDeviceSynchronize(); +#endif +} + + + +/** + * \brief Properties of a given CUDA device and the corresponding PTX bundle + */ +class Device +{ +private: + + /// Type definition of the EmptyKernel kernel entry point + typedef void (*EmptyKernelPtr)(); + + /// Force EmptyKernel to be generated if this class is used + __host__ __device__ __forceinline__ + EmptyKernelPtr Empty() + { + return EmptyKernel; + } + +public: + + // Version information + int sm_version; ///< SM version of target device (SM version X.YZ in XYZ integer form) + int ptx_version; ///< Bundled PTX version for target device (PTX version X.YZ in XYZ integer form) + + // Target device properties + int sm_count; ///< Number of SMs + int warp_threads; ///< Number of threads per warp + int smem_bank_bytes; ///< Number of bytes per SM bank + int smem_banks; ///< Number of smem banks + int smem_bytes; ///< Smem bytes per SM + int smem_alloc_unit; ///< Smem segment size + bool regs_by_block; ///< Whether registers are allocated by threadblock (or by warp) + int reg_alloc_unit; ///< Granularity of register allocation within the SM + int warp_alloc_unit; ///< Granularity of warp allocation within the SM + int max_sm_threads; ///< Maximum number of threads per SM + int max_sm_blocks; ///< Maximum number of threadblocks per SM + int max_block_threads; ///< Maximum number of threads per threadblock + int max_sm_registers; ///< Maximum number of registers per SM + int max_sm_warps; ///< Maximum number of warps per SM + + /** + * Callback for initializing device properties + */ + template + __host__ __device__ __forceinline__ void Callback() + { + warp_threads = ArchProps::WARP_THREADS; + smem_bank_bytes = ArchProps::SMEM_BANK_BYTES; + smem_banks = ArchProps::SMEM_BANKS; + smem_bytes = ArchProps::SMEM_BYTES; + smem_alloc_unit = ArchProps::SMEM_ALLOC_UNIT; + regs_by_block = ArchProps::REGS_BY_BLOCK; + reg_alloc_unit = ArchProps::REG_ALLOC_UNIT; + warp_alloc_unit = ArchProps::WARP_ALLOC_UNIT; + max_sm_threads = ArchProps::MAX_SM_THREADS; + max_sm_blocks = ArchProps::MAX_SM_THREADBLOCKS; + max_block_threads = ArchProps::MAX_BLOCK_THREADS; + max_sm_registers = ArchProps::MAX_SM_REGISTERS; + max_sm_warps = max_sm_threads / warp_threads; + } + + +public: + + /** + * Initializer. Properties are retrieved for the specified GPU ordinal. + */ + __host__ __device__ __forceinline__ + cudaError_t Init(int device_ordinal) + { + #ifndef CUB_RUNTIME_ENABLED + + // CUDA API calls not supported from this device + return CubDebug(cudaErrorInvalidConfiguration); + + #else + + cudaError_t error = cudaSuccess; + do + { + // Fill in SM version + int major, minor; + if (CubDebug(error = cudaDeviceGetAttribute(&major, cudaDevAttrComputeCapabilityMajor, device_ordinal))) break; + if (CubDebug(error = cudaDeviceGetAttribute(&minor, cudaDevAttrComputeCapabilityMinor, device_ordinal))) break; + sm_version = major * 100 + minor * 10; + + // Fill in static SM properties + // Initialize our device properties via callback from static device properties + ArchProps<100>::Callback(*this, sm_version); + + // Fill in SM count + if (CubDebug(error = cudaDeviceGetAttribute (&sm_count, cudaDevAttrMultiProcessorCount, device_ordinal))) break; + + // Fill in PTX version + #if CUB_PTX_ARCH > 0 + ptx_version = CUB_PTX_ARCH; + #else + if (CubDebug(error = PtxVersion(ptx_version))) break; + #endif + + } + while (0); + + return error; + + #endif + } + + + /** + * Initializer. Properties are retrieved for the current GPU ordinal. + */ + __host__ __device__ __forceinline__ + cudaError_t Init() + { + #ifndef CUB_RUNTIME_ENABLED + + // CUDA API calls not supported from this device + return CubDebug(cudaErrorInvalidConfiguration); + + #else + + cudaError_t error = cudaSuccess; + do + { + int device_ordinal; + if ((error = CubDebug(cudaGetDevice(&device_ordinal)))) break; + if ((error = Init(device_ordinal))) break; + } + while (0); + return error; + + #endif + } + + + /** + * Computes maximum SM occupancy in thread blocks for the given kernel + */ + template + __host__ __device__ __forceinline__ + cudaError_t MaxSmOccupancy( + int &max_sm_occupancy, ///< [out] maximum number of thread blocks that can reside on a single SM + KernelPtr kernel_ptr, ///< [in] Kernel pointer for which to compute SM occupancy + int block_threads) ///< [in] Number of threads per thread block + { + #ifndef CUB_RUNTIME_ENABLED + + // CUDA API calls not supported from this device + return CubDebug(cudaErrorInvalidConfiguration); + + #else + + cudaError_t error = cudaSuccess; + do + { + // Get kernel attributes + cudaFuncAttributes kernel_attrs; + if (CubDebug(error = cudaFuncGetAttributes(&kernel_attrs, kernel_ptr))) break; + + // Number of warps per threadblock + int block_warps = (block_threads + warp_threads - 1) / warp_threads; + + // Max warp occupancy + int max_warp_occupancy = (block_warps > 0) ? + max_sm_warps / block_warps : + max_sm_blocks; + + // Maximum register occupancy + int max_reg_occupancy; + if ((block_threads == 0) || (kernel_attrs.numRegs == 0)) + { + // Prevent divide-by-zero + max_reg_occupancy = max_sm_blocks; + } + else if (regs_by_block) + { + // Allocates registers by threadblock + int block_regs = CUB_ROUND_UP_NEAREST(kernel_attrs.numRegs * warp_threads * block_warps, reg_alloc_unit); + max_reg_occupancy = max_sm_registers / block_regs; + } + else + { + // Allocates registers by warp + int sm_sides = warp_alloc_unit; + int sm_registers_per_side = max_sm_registers / sm_sides; + int regs_per_warp = CUB_ROUND_UP_NEAREST(kernel_attrs.numRegs * warp_threads, reg_alloc_unit); + int warps_per_side = sm_registers_per_side / regs_per_warp; + int warps = warps_per_side * sm_sides; + max_reg_occupancy = warps / block_warps; + } + + // Shared memory per threadblock + int block_allocated_smem = CUB_ROUND_UP_NEAREST( + kernel_attrs.sharedSizeBytes, + smem_alloc_unit); + + // Max shared memory occupancy + int max_smem_occupancy = (block_allocated_smem > 0) ? + (smem_bytes / block_allocated_smem) : + max_sm_blocks; + + // Max occupancy + max_sm_occupancy = CUB_MIN( + CUB_MIN(max_sm_blocks, max_warp_occupancy), + CUB_MIN(max_smem_occupancy, max_reg_occupancy)); + +// printf("max_smem_occupancy(%d), max_warp_occupancy(%d), max_reg_occupancy(%d)", max_smem_occupancy, max_warp_occupancy, max_reg_occupancy); + + } while (0); + + return error; + + #endif + } + +}; + + +/** @} */ // end group UtilModule + +} // CUB namespace +CUB_NS_POSTFIX // Optional outer namespace(s) diff --git a/lib/kokkos/TPL/cub/util_iterator.cuh b/lib/kokkos/TPL/cub/util_iterator.cuh new file mode 100755 index 0000000000..08b574ca50 --- /dev/null +++ b/lib/kokkos/TPL/cub/util_iterator.cuh @@ -0,0 +1,718 @@ +/****************************************************************************** + * Copyright (c) 2011, Duane Merrill. All rights reserved. + * Copyright (c) 2011-2013, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the NVIDIA CORPORATION nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + ******************************************************************************/ + +/** + * \file + * Random-access iterator types + */ + +#pragma once + +#include "thread/thread_load.cuh" +#include "util_device.cuh" +#include "util_debug.cuh" +#include "util_namespace.cuh" + +/// Optional outer namespace(s) +CUB_NS_PREFIX + +/// CUB namespace +namespace cub { + + +/****************************************************************************** + * Texture references + *****************************************************************************/ + +#ifndef DOXYGEN_SHOULD_SKIP_THIS // Do not document + +// Anonymous namespace +namespace { + +/// Templated texture reference type +template +struct TexIteratorRef +{ + // Texture reference type + typedef texture TexRef; + + static TexRef ref; + + /** + * Bind texture + */ + static cudaError_t BindTexture(void *d_in) + { + cudaChannelFormatDesc tex_desc = cudaCreateChannelDesc(); + if (d_in) + return (CubDebug(cudaBindTexture(NULL, ref, d_in, tex_desc))); + + return cudaSuccess; + } + + /** + * Unbind textures + */ + static cudaError_t UnbindTexture() + { + return CubDebug(cudaUnbindTexture(ref)); + } +}; + +// Texture reference definitions +template +typename TexIteratorRef::TexRef TexIteratorRef::ref = 0; + +} // Anonymous namespace + + +#endif // DOXYGEN_SHOULD_SKIP_THIS + + + + + + + +/** + * \addtogroup UtilModule + * @{ + */ + + +/****************************************************************************** + * Iterators + *****************************************************************************/ + +/** + * \brief A simple random-access iterator pointing to a range of constant values + * + * \par Overview + * ConstantIteratorRA is a random-access iterator that when dereferenced, always + * returns the supplied constant of type \p OutputType. + * + * \tparam OutputType The value type of this iterator + */ +template +class ConstantIteratorRA +{ +public: + +#ifndef DOXYGEN_SHOULD_SKIP_THIS // Do not document + + typedef ConstantIteratorRA self_type; + typedef OutputType value_type; + typedef OutputType reference; + typedef OutputType* pointer; + typedef std::random_access_iterator_tag iterator_category; + typedef int difference_type; + +#endif // DOXYGEN_SHOULD_SKIP_THIS + +private: + + OutputType val; + +public: + + /// Constructor + __host__ __device__ __forceinline__ ConstantIteratorRA( + const OutputType &val) ///< Constant value for the iterator instance to report + : + val(val) + {} + +#ifndef DOXYGEN_SHOULD_SKIP_THIS // Do not document + + __host__ __device__ __forceinline__ self_type operator++() + { + self_type i = *this; + return i; + } + + __host__ __device__ __forceinline__ self_type operator++(int junk) + { + return *this; + } + + __host__ __device__ __forceinline__ reference operator*() + { + return val; + } + + template + __host__ __device__ __forceinline__ self_type operator+(SizeT n) + { + return ConstantIteratorRA(val); + } + + template + __host__ __device__ __forceinline__ self_type operator-(SizeT n) + { + return ConstantIteratorRA(val); + } + + template + __host__ __device__ __forceinline__ reference operator[](SizeT n) + { + return ConstantIteratorRA(val); + } + + __host__ __device__ __forceinline__ pointer operator->() + { + return &val; + } + + __host__ __device__ __forceinline__ bool operator==(const self_type& rhs) + { + return (val == rhs.val); + } + + __host__ __device__ __forceinline__ bool operator!=(const self_type& rhs) + { + return (val != rhs.val); + } + +#endif // DOXYGEN_SHOULD_SKIP_THIS + +}; + + + +/** + * \brief A simple random-access transform iterator for applying a transformation operator. + * + * \par Overview + * TransformIteratorRA is a random-access iterator that wraps both a native + * device pointer of type InputType* and a unary conversion functor of + * type \p ConversionOp. \p OutputType references are made by pulling \p InputType + * values through the \p ConversionOp instance. + * + * \tparam InputType The value type of the pointer being wrapped + * \tparam ConversionOp Unary functor type for mapping objects of type \p InputType to type \p OutputType. Must have member OutputType operator()(const InputType &datum). + * \tparam OutputType The value type of this iterator + */ +template +class TransformIteratorRA +{ +public: + +#ifndef DOXYGEN_SHOULD_SKIP_THIS // Do not document + + typedef TransformIteratorRA self_type; + typedef OutputType value_type; + typedef OutputType reference; + typedef OutputType* pointer; + typedef std::random_access_iterator_tag iterator_category; + typedef int difference_type; + +#endif // DOXYGEN_SHOULD_SKIP_THIS + +private: + + ConversionOp conversion_op; + InputType* ptr; + +public: + + /** + * \brief Constructor + * @param ptr Native pointer to wrap + * @param conversion_op Binary transformation functor + */ + __host__ __device__ __forceinline__ TransformIteratorRA(InputType* ptr, ConversionOp conversion_op) : + conversion_op(conversion_op), + ptr(ptr) {} + +#ifndef DOXYGEN_SHOULD_SKIP_THIS // Do not document + + __host__ __device__ __forceinline__ self_type operator++() + { + self_type i = *this; + ptr++; + return i; + } + + __host__ __device__ __forceinline__ self_type operator++(int junk) + { + ptr++; + return *this; + } + + __host__ __device__ __forceinline__ reference operator*() + { + return conversion_op(*ptr); + } + + template + __host__ __device__ __forceinline__ self_type operator+(SizeT n) + { + TransformIteratorRA retval(ptr + n, conversion_op); + return retval; + } + + template + __host__ __device__ __forceinline__ self_type operator-(SizeT n) + { + TransformIteratorRA retval(ptr - n, conversion_op); + return retval; + } + + template + __host__ __device__ __forceinline__ reference operator[](SizeT n) + { + return conversion_op(ptr[n]); + } + + __host__ __device__ __forceinline__ pointer operator->() + { + return &conversion_op(*ptr); + } + + __host__ __device__ __forceinline__ bool operator==(const self_type& rhs) + { + return (ptr == rhs.ptr); + } + + __host__ __device__ __forceinline__ bool operator!=(const self_type& rhs) + { + return (ptr != rhs.ptr); + } + +#endif // DOXYGEN_SHOULD_SKIP_THIS + +}; + + + +/** + * \brief A simple random-access iterator for loading primitive values through texture cache. + * + * \par Overview + * TexIteratorRA is a random-access iterator that wraps a native + * device pointer of type T*. References made through TexIteratorRA + * causes values to be pulled through texture cache. + * + * \par Usage Considerations + * - Can only be used with primitive types (e.g., \p char, \p int, \p float), with the exception of \p double + * - Only one TexIteratorRA or TexIteratorRA of a certain \p InputType can be bound at any given time (per host thread) + * + * \tparam InputType The value type of the pointer being wrapped + * \tparam ConversionOp Unary functor type for mapping objects of type \p InputType to type \p OutputType. Must have member OutputType operator()(const InputType &datum). + * \tparam OutputType The value type of this iterator + */ +template +class TexIteratorRA +{ +public: +#ifndef DOXYGEN_SHOULD_SKIP_THIS // Do not document + + typedef TexIteratorRA self_type; + typedef T value_type; + typedef T reference; + typedef T* pointer; + typedef std::random_access_iterator_tag iterator_category; + typedef int difference_type; + +#endif // DOXYGEN_SHOULD_SKIP_THIS + + /// Tag identifying iterator type as being texture-bindable + typedef void TexBindingTag; + +private: + + T* ptr; + size_t tex_align_offset; + cudaTextureObject_t tex_obj; + +public: + + /** + * \brief Constructor + */ + __host__ __device__ __forceinline__ TexIteratorRA() + : + ptr(NULL), + tex_align_offset(0), + tex_obj(0) + {} + + /// \brief Bind iterator to texture reference + cudaError_t BindTexture( + T *ptr, ///< Native pointer to wrap that is aligned to cudaDeviceProp::textureAlignment + size_t bytes, ///< Number of items + size_t tex_align_offset = 0) ///< Offset (in items) from ptr denoting the position of the iterator + { + this->ptr = ptr; + this->tex_align_offset = tex_align_offset; + + int ptx_version; + cudaError_t error = cudaSuccess; + if (CubDebug(error = PtxVersion(ptx_version))) return error; + if (ptx_version >= 300) + { + // Use texture object + cudaChannelFormatDesc channel_desc = cudaCreateChannelDesc(); + cudaResourceDesc res_desc; + cudaTextureDesc tex_desc; + memset(&res_desc, 0, sizeof(cudaResourceDesc)); + memset(&tex_desc, 0, sizeof(cudaTextureDesc)); + res_desc.resType = cudaResourceTypeLinear; + res_desc.res.linear.devPtr = ptr; + res_desc.res.linear.desc = channel_desc; + res_desc.res.linear.sizeInBytes = bytes; + tex_desc.readMode = cudaReadModeElementType; + return cudaCreateTextureObject(&tex_obj, &res_desc, &tex_desc, NULL); + } + else + { + // Use texture reference + return TexIteratorRef::BindTexture(ptr); + } + } + + /// \brief Unbind iterator to texture reference + cudaError_t UnbindTexture() + { + int ptx_version; + cudaError_t error = cudaSuccess; + if (CubDebug(error = PtxVersion(ptx_version))) return error; + if (ptx_version < 300) + { + // Use texture reference + return TexIteratorRef::UnbindTexture(); + } + else + { + // Use texture object + return cudaDestroyTextureObject(tex_obj); + } + } + +#ifndef DOXYGEN_SHOULD_SKIP_THIS // Do not document + + __host__ __device__ __forceinline__ self_type operator++() + { + self_type i = *this; + ptr++; + tex_align_offset++; + return i; + } + + __host__ __device__ __forceinline__ self_type operator++(int junk) + { + ptr++; + tex_align_offset++; + return *this; + } + + __host__ __device__ __forceinline__ reference operator*() + { +#if (CUB_PTX_ARCH == 0) + // Simply dereference the pointer on the host + return *ptr; +#elif (CUB_PTX_ARCH < 300) + // Use the texture reference + return tex1Dfetch(TexIteratorRef::ref, tex_align_offset); +#else + // Use the texture object + return conversion_op(tex1Dfetch(tex_obj, tex_align_offset)); +#endif + } + + template + __host__ __device__ __forceinline__ self_type operator+(SizeT n) + { + TexIteratorRA retval; + retval.ptr = ptr + n; + retval.tex_align_offset = tex_align_offset + n; + return retval; + } + + template + __host__ __device__ __forceinline__ self_type operator-(SizeT n) + { + TexIteratorRA retval; + retval.ptr = ptr - n; + retval.tex_align_offset = tex_align_offset - n; + return retval; + } + + template + __host__ __device__ __forceinline__ reference operator[](SizeT n) + { +#if (CUB_PTX_ARCH == 0) + // Simply dereference the pointer on the host + return ptr[n]; +#elif (CUB_PTX_ARCH < 300) + // Use the texture reference + return tex1Dfetch(TexIteratorRef::ref, tex_align_offset + n); +#else + // Use the texture object + return conversion_op(tex1Dfetch(tex_obj, tex_align_offset + n)); +#endif + } + + __host__ __device__ __forceinline__ pointer operator->() + { +#if (CUB_PTX_ARCH == 0) + // Simply dereference the pointer on the host + return &(*ptr); +#elif (CUB_PTX_ARCH < 300) + // Use the texture reference + return &(tex1Dfetch(TexIteratorRef::ref, tex_align_offset)); +#else + // Use the texture object + return conversion_op(tex1Dfetch(tex_obj, tex_align_offset)); +#endif + } + + __host__ __device__ __forceinline__ bool operator==(const self_type& rhs) + { + return (ptr == rhs.ptr); + } + + __host__ __device__ __forceinline__ bool operator!=(const self_type& rhs) + { + return (ptr != rhs.ptr); + } + +#endif // DOXYGEN_SHOULD_SKIP_THIS + +}; + + +/** + * \brief A simple random-access transform iterator for loading primitive values through texture cache and and subsequently applying a transformation operator. + * + * \par Overview + * TexTransformIteratorRA is a random-access iterator that wraps both a native + * device pointer of type InputType* and a unary conversion functor of + * type \p ConversionOp. \p OutputType references are made by pulling \p InputType + * values through the texture cache and then transformed them using the + * \p ConversionOp instance. + * + * \par Usage Considerations + * - Can only be used with primitive types (e.g., \p char, \p int, \p float), with the exception of \p double + * - Only one TexIteratorRA or TexTransformIteratorRA of a certain \p InputType can be bound at any given time (per host thread) + * + * \tparam InputType The value type of the pointer being wrapped + * \tparam ConversionOp Unary functor type for mapping objects of type \p InputType to type \p OutputType. Must have member OutputType operator()(const InputType &datum). + * \tparam OutputType The value type of this iterator + */ +template +class TexTransformIteratorRA +{ +public: + +#ifndef DOXYGEN_SHOULD_SKIP_THIS // Do not document + + typedef TexTransformIteratorRA self_type; + typedef OutputType value_type; + typedef OutputType reference; + typedef OutputType* pointer; + typedef std::random_access_iterator_tag iterator_category; + typedef int difference_type; + +#endif // DOXYGEN_SHOULD_SKIP_THIS + + /// Tag identifying iterator type as being texture-bindable + typedef void TexBindingTag; + +private: + + ConversionOp conversion_op; + InputType* ptr; + size_t tex_align_offset; + cudaTextureObject_t tex_obj; + +public: + + /** + * \brief Constructor + */ + TexTransformIteratorRA( + ConversionOp conversion_op) ///< Binary transformation functor + : + conversion_op(conversion_op), + ptr(NULL), + tex_align_offset(0), + tex_obj(0) + {} + + /// \brief Bind iterator to texture reference + cudaError_t BindTexture( + InputType* ptr, ///< Native pointer to wrap that is aligned to cudaDeviceProp::textureAlignment + size_t bytes, ///< Number of items + size_t tex_align_offset = 0) ///< Offset (in items) from ptr denoting the position of the iterator + { + this->ptr = ptr; + this->tex_align_offset = tex_align_offset; + + int ptx_version; + cudaError_t error = cudaSuccess; + if (CubDebug(error = PtxVersion(ptx_version))) return error; + if (ptx_version >= 300) + { + // Use texture object + cudaChannelFormatDesc channel_desc = cudaCreateChannelDesc(); + cudaResourceDesc res_desc; + cudaTextureDesc tex_desc; + memset(&res_desc, 0, sizeof(cudaResourceDesc)); + memset(&tex_desc, 0, sizeof(cudaTextureDesc)); + res_desc.resType = cudaResourceTypeLinear; + res_desc.res.linear.devPtr = ptr; + res_desc.res.linear.desc = channel_desc; + res_desc.res.linear.sizeInBytes = bytes; + tex_desc.readMode = cudaReadModeElementType; + return cudaCreateTextureObject(&tex_obj, &res_desc, &tex_desc, NULL); + } + else + { + // Use texture reference + return TexIteratorRef::BindTexture(ptr); + } + } + + /// \brief Unbind iterator to texture reference + cudaError_t UnbindTexture() + { + int ptx_version; + cudaError_t error = cudaSuccess; + if (CubDebug(error = PtxVersion(ptx_version))) return error; + if (ptx_version >= 300) + { + // Use texture object + return cudaDestroyTextureObject(tex_obj); + } + else + { + // Use texture reference + return TexIteratorRef::UnbindTexture(); + } + } + +#ifndef DOXYGEN_SHOULD_SKIP_THIS // Do not document + + __host__ __device__ __forceinline__ self_type operator++() + { + self_type i = *this; + ptr++; + tex_align_offset++; + return i; + } + + __host__ __device__ __forceinline__ self_type operator++(int junk) + { + ptr++; + tex_align_offset++; + return *this; + } + + __host__ __device__ __forceinline__ reference operator*() + { +#if (CUB_PTX_ARCH == 0) + // Simply dereference the pointer on the host + return conversion_op(*ptr); +#elif (CUB_PTX_ARCH < 300) + // Use the texture reference + return conversion_op(tex1Dfetch(TexIteratorRef::ref, tex_align_offset)); +#else + // Use the texture object + return conversion_op(tex1Dfetch(tex_obj, tex_align_offset)); +#endif + } + + template + __host__ __device__ __forceinline__ self_type operator+(SizeT n) + { + TexTransformIteratorRA retval(conversion_op); + retval.ptr = ptr + n; + retval.tex_align_offset = tex_align_offset + n; + return retval; + } + + template + __host__ __device__ __forceinline__ self_type operator-(SizeT n) + { + TexTransformIteratorRA retval(conversion_op); + retval.ptr = ptr - n; + retval.tex_align_offset = tex_align_offset - n; + return retval; + } + + template + __host__ __device__ __forceinline__ reference operator[](SizeT n) + { +#if (CUB_PTX_ARCH == 0) + // Simply dereference the pointer on the host + return conversion_op(ptr[n]); +#elif (CUB_PTX_ARCH < 300) + // Use the texture reference + return conversion_op(tex1Dfetch(TexIteratorRef::ref, tex_align_offset + n)); +#else + // Use the texture object + return conversion_op(tex1Dfetch(tex_obj, tex_align_offset + n)); +#endif + } + + __host__ __device__ __forceinline__ pointer operator->() + { +#if (CUB_PTX_ARCH == 0) + // Simply dereference the pointer on the host + return &conversion_op(*ptr); +#elif (CUB_PTX_ARCH < 300) + // Use the texture reference + return &conversion_op(tex1Dfetch(TexIteratorRef::ref, tex_align_offset)); +#else + // Use the texture object + return &conversion_op(tex1Dfetch(tex_obj, tex_align_offset)); +#endif + } + + __host__ __device__ __forceinline__ bool operator==(const self_type& rhs) + { + return (ptr == rhs.ptr); + } + + __host__ __device__ __forceinline__ bool operator!=(const self_type& rhs) + { + return (ptr != rhs.ptr); + } + +#endif // DOXYGEN_SHOULD_SKIP_THIS + +}; + + + + +/** @} */ // end group UtilModule + +} // CUB namespace +CUB_NS_POSTFIX // Optional outer namespace(s) diff --git a/lib/kokkos/TPL/cub/util_macro.cuh b/lib/kokkos/TPL/cub/util_macro.cuh new file mode 100755 index 0000000000..091fd93c55 --- /dev/null +++ b/lib/kokkos/TPL/cub/util_macro.cuh @@ -0,0 +1,107 @@ +/****************************************************************************** + * Copyright (c) 2011, Duane Merrill. All rights reserved. + * Copyright (c) 2011-2013, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the NVIDIA CORPORATION nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + ******************************************************************************/ + +/****************************************************************************** + * Common C/C++ macro utilities + ******************************************************************************/ + +#pragma once + +#include "util_namespace.cuh" + +/// Optional outer namespace(s) +CUB_NS_PREFIX + +/// CUB namespace +namespace cub { + + +/** + * \addtogroup UtilModule + * @{ + */ + +/** + * Align struct + */ +#if defined(_WIN32) || defined(_WIN64) + #define CUB_ALIGN(bytes) __declspec(align(32)) +#else + #define CUB_ALIGN(bytes) __attribute__((aligned(bytes))) +#endif + +/** + * Select maximum(a, b) + */ +#define CUB_MAX(a, b) (((a) > (b)) ? (a) : (b)) + +/** + * Select minimum(a, b) + */ +#define CUB_MIN(a, b) (((a) < (b)) ? (a) : (b)) + +/** + * Quotient of x/y rounded down to nearest integer + */ +#define CUB_QUOTIENT_FLOOR(x, y) ((x) / (y)) + +/** + * Quotient of x/y rounded up to nearest integer + */ +#define CUB_QUOTIENT_CEILING(x, y) (((x) + (y) - 1) / (y)) + +/** + * x rounded up to the nearest multiple of y + */ +#define CUB_ROUND_UP_NEAREST(x, y) ((((x) + (y) - 1) / (y)) * y) + +/** + * x rounded down to the nearest multiple of y + */ +#define CUB_ROUND_DOWN_NEAREST(x, y) (((x) / (y)) * y) + +/** + * Return character string for given type + */ +#define CUB_TYPE_STRING(type) ""#type + +#ifndef DOXYGEN_SHOULD_SKIP_THIS // Do not document + #define CUB_CAT_(a, b) a ## b + #define CUB_CAT(a, b) CUB_CAT_(a, b) +#endif // DOXYGEN_SHOULD_SKIP_THIS + +/** + * Static assert + */ +#define CUB_STATIC_ASSERT(cond, msg) typedef int CUB_CAT(cub_static_assert, __LINE__)[(cond) ? 1 : -1] + + +/** @} */ // end group UtilModule + +} // CUB namespace +CUB_NS_POSTFIX // Optional outer namespace(s) diff --git a/lib/kokkos/TPL/cub/util_namespace.cuh b/lib/kokkos/TPL/cub/util_namespace.cuh new file mode 100755 index 0000000000..869ecc613a --- /dev/null +++ b/lib/kokkos/TPL/cub/util_namespace.cuh @@ -0,0 +1,41 @@ +/****************************************************************************** + * Copyright (c) 2011, Duane Merrill. All rights reserved. + * Copyright (c) 2011-2013, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the NVIDIA CORPORATION nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + ******************************************************************************/ + +/** + * \file + * Place-holder for prefixing the cub namespace + */ + +#pragma once + +// For example: +//#define CUB_NS_PREFIX namespace thrust{ namespace detail { +//#define CUB_NS_POSTFIX } } + +#define CUB_NS_PREFIX +#define CUB_NS_POSTFIX diff --git a/lib/kokkos/TPL/cub/util_ptx.cuh b/lib/kokkos/TPL/cub/util_ptx.cuh new file mode 100755 index 0000000000..ad80b04016 --- /dev/null +++ b/lib/kokkos/TPL/cub/util_ptx.cuh @@ -0,0 +1,380 @@ +/****************************************************************************** + * Copyright (c) 2011, Duane Merrill. All rights reserved. + * Copyright (c) 2011-2013, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the NVIDIA CORPORATION nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + ******************************************************************************/ + +/** + * \file + * PTX intrinsics + */ + + +#pragma once + +#include "util_type.cuh" +#include "util_arch.cuh" +#include "util_namespace.cuh" + +/// Optional outer namespace(s) +CUB_NS_PREFIX + +/// CUB namespace +namespace cub { + + +/** + * \addtogroup UtilModule + * @{ + */ + + +/****************************************************************************** + * PTX helper macros + ******************************************************************************/ + +/** + * Register modifier for pointer-types (for inlining PTX assembly) + */ +#if defined(_WIN64) || defined(__LP64__) + #define __CUB_LP64__ 1 + // 64-bit register modifier for inlined asm + #define _CUB_ASM_PTR_ "l" + #define _CUB_ASM_PTR_SIZE_ "u64" +#else + #define __CUB_LP64__ 0 + // 32-bit register modifier for inlined asm + #define _CUB_ASM_PTR_ "r" + #define _CUB_ASM_PTR_SIZE_ "u32" +#endif + + +/****************************************************************************** + * Inlined PTX intrinsics + ******************************************************************************/ + +/** + * Shift-right then add. Returns (x >> shift) + addend. + */ +__device__ __forceinline__ unsigned int SHR_ADD( + unsigned int x, + unsigned int shift, + unsigned int addend) +{ + unsigned int ret; +#if __CUDA_ARCH__ >= 200 + asm("vshr.u32.u32.u32.clamp.add %0, %1, %2, %3;" : + "=r"(ret) : "r"(x), "r"(shift), "r"(addend)); +#else + ret = (x >> shift) + addend; +#endif + return ret; +} + + +/** + * Shift-left then add. Returns (x << shift) + addend. + */ +__device__ __forceinline__ unsigned int SHL_ADD( + unsigned int x, + unsigned int shift, + unsigned int addend) +{ + unsigned int ret; +#if __CUDA_ARCH__ >= 200 + asm("vshl.u32.u32.u32.clamp.add %0, %1, %2, %3;" : + "=r"(ret) : "r"(x), "r"(shift), "r"(addend)); +#else + ret = (x << shift) + addend; +#endif + return ret; +} + + +/** + * Bitfield-extract. + */ +template +__device__ __forceinline__ unsigned int BFE( + UnsignedBits source, + unsigned int bit_start, + unsigned int num_bits) +{ + unsigned int bits; +#if __CUDA_ARCH__ >= 200 + asm("bfe.u32 %0, %1, %2, %3;" : "=r"(bits) : "r"((unsigned int) source), "r"(bit_start), "r"(num_bits)); +#else + const unsigned int MASK = (1 << num_bits) - 1; + bits = (source >> bit_start) & MASK; +#endif + return bits; +} + + +/** + * Bitfield-extract for 64-bit types. + */ +__device__ __forceinline__ unsigned int BFE( + unsigned long long source, + unsigned int bit_start, + unsigned int num_bits) +{ + const unsigned long long MASK = (1ull << num_bits) - 1; + return (source >> bit_start) & MASK; +} + + +/** + * Bitfield insert. Inserts the first num_bits of y into x starting at bit_start + */ +__device__ __forceinline__ void BFI( + unsigned int &ret, + unsigned int x, + unsigned int y, + unsigned int bit_start, + unsigned int num_bits) +{ +#if __CUDA_ARCH__ >= 200 + asm("bfi.b32 %0, %1, %2, %3, %4;" : + "=r"(ret) : "r"(y), "r"(x), "r"(bit_start), "r"(num_bits)); +#else + // TODO +#endif +} + + +/** + * Three-operand add + */ +__device__ __forceinline__ unsigned int IADD3(unsigned int x, unsigned int y, unsigned int z) +{ +#if __CUDA_ARCH__ >= 200 + asm("vadd.u32.u32.u32.add %0, %1, %2, %3;" : "=r"(x) : "r"(x), "r"(y), "r"(z)); +#else + x = x + y + z; +#endif + return x; +} + + +/** + * Byte-permute. Pick four arbitrary bytes from two 32-bit registers, and + * reassemble them into a 32-bit destination register + */ +__device__ __forceinline__ int PRMT(unsigned int a, unsigned int b, unsigned int index) +{ + int ret; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(ret) : "r"(a), "r"(b), "r"(index)); + return ret; +} + + +/** + * Sync-threads barrier. + */ +__device__ __forceinline__ void BAR(int count) +{ + asm volatile("bar.sync 1, %0;" : : "r"(count)); +} + + +/** + * Floating point multiply. (Mantissa LSB rounds towards zero.) + */ +__device__ __forceinline__ float FMUL_RZ(float a, float b) +{ + float d; + asm("mul.rz.f32 %0, %1, %2;" : "=f"(d) : "f"(a), "f"(b)); + return d; +} + + +/** + * Floating point multiply-add. (Mantissa LSB rounds towards zero.) + */ +__device__ __forceinline__ float FFMA_RZ(float a, float b, float c) +{ + float d; + asm("fma.rz.f32 %0, %1, %2, %3;" : "=f"(d) : "f"(a), "f"(b), "f"(c)); + return d; +} + + +/** + * Terminates the calling thread + */ +__device__ __forceinline__ void ThreadExit() { + asm("exit;"); +} + + +/** + * Returns the warp lane ID of the calling thread + */ +__device__ __forceinline__ unsigned int LaneId() +{ + unsigned int ret; + asm("mov.u32 %0, %laneid;" : "=r"(ret) ); + return ret; +} + + +/** + * Returns the warp ID of the calling thread + */ +__device__ __forceinline__ unsigned int WarpId() +{ + unsigned int ret; + asm("mov.u32 %0, %warpid;" : "=r"(ret) ); + return ret; +} + +/** + * Returns the warp lane mask of all lanes less than the calling thread + */ +__device__ __forceinline__ unsigned int LaneMaskLt() +{ + unsigned int ret; + asm("mov.u32 %0, %lanemask_lt;" : "=r"(ret) ); + return ret; +} + +/** + * Returns the warp lane mask of all lanes less than or equal to the calling thread + */ +__device__ __forceinline__ unsigned int LaneMaskLe() +{ + unsigned int ret; + asm("mov.u32 %0, %lanemask_le;" : "=r"(ret) ); + return ret; +} + +/** + * Returns the warp lane mask of all lanes greater than the calling thread + */ +__device__ __forceinline__ unsigned int LaneMaskGt() +{ + unsigned int ret; + asm("mov.u32 %0, %lanemask_gt;" : "=r"(ret) ); + return ret; +} + +/** + * Returns the warp lane mask of all lanes greater than or equal to the calling thread + */ +__device__ __forceinline__ unsigned int LaneMaskGe() +{ + unsigned int ret; + asm("mov.u32 %0, %lanemask_ge;" : "=r"(ret) ); + return ret; +} + +/** + * Portable implementation of __all + */ +__device__ __forceinline__ int WarpAll(int cond) +{ +#if CUB_PTX_ARCH < 120 + + __shared__ volatile int warp_signals[PtxArchProps::MAX_SM_THREADS / PtxArchProps::WARP_THREADS]; + + if (LaneId() == 0) + warp_signals[WarpId()] = 1; + + if (cond == 0) + warp_signals[WarpId()] = 0; + + return warp_signals[WarpId()]; + +#else + + return __all(cond); + +#endif +} + + +/** + * Portable implementation of __any + */ +__device__ __forceinline__ int WarpAny(int cond) +{ +#if CUB_PTX_ARCH < 120 + + __shared__ volatile int warp_signals[PtxArchProps::MAX_SM_THREADS / PtxArchProps::WARP_THREADS]; + + if (LaneId() == 0) + warp_signals[WarpId()] = 0; + + if (cond) + warp_signals[WarpId()] = 1; + + return warp_signals[WarpId()]; + +#else + + return __any(cond); + +#endif +} + + +/// Generic shuffle-up +template +__device__ __forceinline__ T ShuffleUp( + T input, ///< [in] The value to broadcast + int src_offset) ///< [in] The up-offset of the peer to read from +{ + enum + { + SHFL_C = 0, + }; + + typedef typename WordAlignment::ShuffleWord ShuffleWord; + + const int WORDS = (sizeof(T) + sizeof(ShuffleWord) - 1) / sizeof(ShuffleWord); + T output; + ShuffleWord *output_alias = reinterpret_cast(&output); + ShuffleWord *input_alias = reinterpret_cast(&input); + + #pragma unroll + for (int WORD = 0; WORD < WORDS; ++WORD) + { + unsigned int shuffle_word = input_alias[WORD]; + asm( + " shfl.up.b32 %0, %1, %2, %3;" + : "=r"(shuffle_word) : "r"(shuffle_word), "r"(src_offset), "r"(SHFL_C)); + output_alias[WORD] = (ShuffleWord) shuffle_word; + } + + return output; +} + + + +/** @} */ // end group UtilModule + +} // CUB namespace +CUB_NS_POSTFIX // Optional outer namespace(s) diff --git a/lib/kokkos/TPL/cub/util_type.cuh b/lib/kokkos/TPL/cub/util_type.cuh new file mode 100755 index 0000000000..836aa0f043 --- /dev/null +++ b/lib/kokkos/TPL/cub/util_type.cuh @@ -0,0 +1,685 @@ +/****************************************************************************** + * Copyright (c) 2011, Duane Merrill. All rights reserved. + * Copyright (c) 2011-2013, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the NVIDIA CORPORATION nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + ******************************************************************************/ + +/** + * \file + * Common type manipulation (metaprogramming) utilities + */ + +#pragma once + +#include +#include + +#include "util_namespace.cuh" + +/// Optional outer namespace(s) +CUB_NS_PREFIX + +/// CUB namespace +namespace cub { + + +/** + * \addtogroup UtilModule + * @{ + */ + + + +/****************************************************************************** + * Type equality + ******************************************************************************/ + +/** + * \brief Type selection (IF ? ThenType : ElseType) + */ +template +struct If +{ + /// Conditional type result + typedef ThenType Type; // true +}; + +#ifndef DOXYGEN_SHOULD_SKIP_THIS // Do not document + +template +struct If +{ + typedef ElseType Type; // false +}; + +#endif // DOXYGEN_SHOULD_SKIP_THIS + + +/****************************************************************************** + * Conditional types + ******************************************************************************/ + + +/** + * \brief Type equality test + */ +template +struct Equals +{ + enum { + VALUE = 0, + NEGATE = 1 + }; +}; + +#ifndef DOXYGEN_SHOULD_SKIP_THIS // Do not document + +template +struct Equals +{ + enum { + VALUE = 1, + NEGATE = 0 + }; +}; + +#endif // DOXYGEN_SHOULD_SKIP_THIS + + +/****************************************************************************** + * Marker types + ******************************************************************************/ + +/** + * \brief A simple "NULL" marker type + */ +struct NullType +{ +#ifndef DOXYGEN_SHOULD_SKIP_THIS // Do not document + template + __host__ __device__ __forceinline__ NullType& operator =(const T& b) { return *this; } +#endif // DOXYGEN_SHOULD_SKIP_THIS +}; + + +/** + * \brief Allows for the treatment of an integral constant as a type at compile-time (e.g., to achieve static call dispatch based on constant integral values) + */ +template +struct Int2Type +{ + enum {VALUE = A}; +}; + + +/****************************************************************************** + * Size and alignment + ******************************************************************************/ + +#ifndef DOXYGEN_SHOULD_SKIP_THIS // Do not document + +template +struct WordAlignment +{ + struct Pad + { + T val; + char byte; + }; + + enum + { + /// The alignment of T in bytes + ALIGN_BYTES = sizeof(Pad) - sizeof(T) + }; + + /// Biggest shuffle word that T is a whole multiple of and is not larger than the alignment of T + typedef typename If<(ALIGN_BYTES % 4 == 0), + int, + typename If<(ALIGN_BYTES % 2 == 0), + short, + char>::Type>::Type ShuffleWord; + + /// Biggest volatile word that T is a whole multiple of and is not larger than the alignment of T + typedef typename If<(ALIGN_BYTES % 8 == 0), + long long, + ShuffleWord>::Type VolatileWord; + + /// Biggest memory-access word that T is a whole multiple of and is not larger than the alignment of T + typedef typename If<(ALIGN_BYTES % 16 == 0), + longlong2, + typename If<(ALIGN_BYTES % 8 == 0), + long long, // needed to get heterogenous PODs to work on all platforms + ShuffleWord>::Type>::Type DeviceWord; + + enum + { + DEVICE_MULTIPLE = sizeof(DeviceWord) / sizeof(T) + }; + + struct UninitializedBytes + { + char buf[sizeof(T)]; + }; + + struct UninitializedShuffleWords + { + ShuffleWord buf[sizeof(T) / sizeof(ShuffleWord)]; + }; + + struct UninitializedVolatileWords + { + VolatileWord buf[sizeof(T) / sizeof(VolatileWord)]; + }; + + struct UninitializedDeviceWords + { + DeviceWord buf[sizeof(T) / sizeof(DeviceWord)]; + }; + + +}; + + +#endif // DOXYGEN_SHOULD_SKIP_THIS + + +/****************************************************************************** + * Wrapper types + ******************************************************************************/ + +/** + * \brief A storage-backing wrapper that allows types with non-trivial constructors to be aliased in unions + */ +template +struct Uninitialized +{ + /// Biggest memory-access word that T is a whole multiple of and is not larger than the alignment of T + typedef typename WordAlignment::DeviceWord DeviceWord; + + enum + { + WORDS = sizeof(T) / sizeof(DeviceWord) + }; + + /// Backing storage + DeviceWord storage[WORDS]; + + /// Alias + __host__ __device__ __forceinline__ T& Alias() + { + return reinterpret_cast(*this); + } +}; + + +/** + * \brief A wrapper for passing simple static arrays as kernel parameters + */ +template +struct ArrayWrapper +{ + /// Static array of type \p T + T array[COUNT]; +}; + + +/** + * \brief Double-buffer storage wrapper for multi-pass stream transformations that require more than one storage array for streaming intermediate results back and forth. + * + * Many multi-pass computations require a pair of "ping-pong" storage + * buffers (e.g., one for reading from and the other for writing to, and then + * vice-versa for the subsequent pass). This structure wraps a set of device + * buffers and a "selector" member to track which is "current". + */ +template +struct DoubleBuffer +{ + /// Pair of device buffer pointers + T *d_buffers[2]; + + /// Selector into \p d_buffers (i.e., the active/valid buffer) + int selector; + + /// \brief Constructor + __host__ __device__ __forceinline__ DoubleBuffer() + { + selector = 0; + d_buffers[0] = NULL; + d_buffers[1] = NULL; + } + + /// \brief Constructor + __host__ __device__ __forceinline__ DoubleBuffer( + T *d_current, ///< The currently valid buffer + T *d_alternate) ///< Alternate storage buffer of the same size as \p d_current + { + selector = 0; + d_buffers[0] = d_current; + d_buffers[1] = d_alternate; + } + + /// \brief Return pointer to the currently valid buffer + __host__ __device__ __forceinline__ T* Current() { return d_buffers[selector]; } +}; + + + +/****************************************************************************** + * Static math + ******************************************************************************/ + +/** + * \brief Statically determine log2(N), rounded up. + * + * For example: + * Log2<8>::VALUE // 3 + * Log2<3>::VALUE // 2 + */ +template +struct Log2 +{ + /// Static logarithm value + enum { VALUE = Log2> 1), COUNT + 1>::VALUE }; // Inductive case +}; + +#ifndef DOXYGEN_SHOULD_SKIP_THIS // Do not document +template +struct Log2 +{ + enum {VALUE = (1 << (COUNT - 1) < N) ? // Base case + COUNT : + COUNT - 1 }; +}; +#endif // DOXYGEN_SHOULD_SKIP_THIS + + +/** + * \brief Statically determine if N is a power-of-two + */ +template +struct PowerOfTwo +{ + enum { VALUE = ((N & (N - 1)) == 0) }; +}; + + + +/****************************************************************************** + * Pointer vs. iterator detection + ******************************************************************************/ + + +/** + * \brief Pointer vs. iterator + */ +template +struct IsPointer +{ + enum { VALUE = 0 }; +}; + +#ifndef DOXYGEN_SHOULD_SKIP_THIS // Do not document + +template +struct IsPointer +{ + enum { VALUE = 1 }; +}; + +#endif // DOXYGEN_SHOULD_SKIP_THIS + + + +/****************************************************************************** + * Qualifier detection + ******************************************************************************/ + +/** + * \brief Volatile modifier test + */ +template +struct IsVolatile +{ + enum { VALUE = 0 }; +}; + +#ifndef DOXYGEN_SHOULD_SKIP_THIS // Do not document + +template +struct IsVolatile +{ + enum { VALUE = 1 }; +}; + +#endif // DOXYGEN_SHOULD_SKIP_THIS + + +/****************************************************************************** + * Qualifier removal + ******************************************************************************/ + +/** + * \brief Removes \p const and \p volatile qualifiers from type \p Tp. + * + * For example: + * typename RemoveQualifiers::Type // int; + */ +template +struct RemoveQualifiers +{ + /// Type without \p const and \p volatile qualifiers + typedef Up Type; +}; + +#ifndef DOXYGEN_SHOULD_SKIP_THIS // Do not document + +template +struct RemoveQualifiers +{ + typedef Up Type; +}; + +template +struct RemoveQualifiers +{ + typedef Up Type; +}; + +template +struct RemoveQualifiers +{ + typedef Up Type; +}; + +#endif // DOXYGEN_SHOULD_SKIP_THIS + + + +/****************************************************************************** + * Typedef-detection + ******************************************************************************/ + + +/** + * \brief Defines a structure \p detector_name that is templated on type \p T. The \p detector_name struct exposes a constant member \p VALUE indicating whether or not parameter \p T exposes a nested type \p nested_type_name + */ +#define CUB_DEFINE_DETECT_NESTED_TYPE(detector_name, nested_type_name) \ + template \ + struct detector_name \ + { \ + template \ + static char& test(typename C::nested_type_name*); \ + template \ + static int& test(...); \ + enum \ + { \ + VALUE = sizeof(test(0)) < sizeof(int) \ + }; \ + }; + + + +/****************************************************************************** + * Simple enable-if (similar to Boost) + ******************************************************************************/ + +/** + * \brief Simple enable-if (similar to Boost) + */ +template +struct EnableIf +{ + /// Enable-if type for SFINAE dummy variables + typedef T Type; +}; + +#ifndef DOXYGEN_SHOULD_SKIP_THIS // Do not document + +template +struct EnableIf {}; + +#endif // DOXYGEN_SHOULD_SKIP_THIS + + +/****************************************************************************** + * Typedef-detection + ******************************************************************************/ + +/** + * \brief Determine whether or not BinaryOp's functor is of the form bool operator()(const T& a, const T&b) or bool operator()(const T& a, const T&b, unsigned int idx) + */ +template +struct BinaryOpHasIdxParam +{ +private: + template struct SFINAE1 {}; + template struct SFINAE2 {}; + template struct SFINAE3 {}; + template struct SFINAE4 {}; + + template struct SFINAE5 {}; + template struct SFINAE6 {}; + template struct SFINAE7 {}; + template struct SFINAE8 {}; + + template static char Test(SFINAE1 *); + template static char Test(SFINAE2 *); + template static char Test(SFINAE3 *); + template static char Test(SFINAE4 *); + + template static char Test(SFINAE5 *); + template static char Test(SFINAE6 *); + template static char Test(SFINAE7 *); + template static char Test(SFINAE8 *); + + template static int Test(...); + +public: + + /// Whether the functor BinaryOp has a third unsigned int index param + static const bool HAS_PARAM = sizeof(Test(NULL)) == sizeof(char); +}; + + + +/****************************************************************************** + * Simple type traits utilities. + * + * For example: + * Traits::CATEGORY // SIGNED_INTEGER + * Traits::NULL_TYPE // true + * Traits::CATEGORY // NOT_A_NUMBER + * Traits::PRIMITIVE; // false + * + ******************************************************************************/ + +/** + * \brief Basic type traits categories + */ +enum Category +{ + NOT_A_NUMBER, + SIGNED_INTEGER, + UNSIGNED_INTEGER, + FLOATING_POINT +}; + + +/** + * \brief Basic type traits + */ +template +struct BaseTraits +{ + /// Category + static const Category CATEGORY = _CATEGORY; + enum + { + PRIMITIVE = _PRIMITIVE, + NULL_TYPE = _NULL_TYPE, + }; +}; + +#ifndef DOXYGEN_SHOULD_SKIP_THIS // Do not document + +/** + * Basic type traits (unsigned primitive specialization) + */ +template +struct BaseTraits +{ + typedef _UnsignedBits UnsignedBits; + + static const Category CATEGORY = UNSIGNED_INTEGER; + static const UnsignedBits MIN_KEY = UnsignedBits(0); + static const UnsignedBits MAX_KEY = UnsignedBits(-1); + + enum + { + PRIMITIVE = true, + NULL_TYPE = false, + }; + + + static __device__ __forceinline__ UnsignedBits TwiddleIn(UnsignedBits key) + { + return key; + } + + static __device__ __forceinline__ UnsignedBits TwiddleOut(UnsignedBits key) + { + return key; + } +}; + + +/** + * Basic type traits (signed primitive specialization) + */ +template +struct BaseTraits +{ + typedef _UnsignedBits UnsignedBits; + + static const Category CATEGORY = SIGNED_INTEGER; + static const UnsignedBits HIGH_BIT = UnsignedBits(1) << ((sizeof(UnsignedBits) * 8) - 1); + static const UnsignedBits MIN_KEY = HIGH_BIT; + static const UnsignedBits MAX_KEY = UnsignedBits(-1) ^ HIGH_BIT; + + enum + { + PRIMITIVE = true, + NULL_TYPE = false, + }; + + static __device__ __forceinline__ UnsignedBits TwiddleIn(UnsignedBits key) + { + return key ^ HIGH_BIT; + }; + + static __device__ __forceinline__ UnsignedBits TwiddleOut(UnsignedBits key) + { + return key ^ HIGH_BIT; + }; + +}; + + +/** + * Basic type traits (fp primitive specialization) + */ +template +struct BaseTraits +{ + typedef _UnsignedBits UnsignedBits; + + static const Category CATEGORY = FLOATING_POINT; + static const UnsignedBits HIGH_BIT = UnsignedBits(1) << ((sizeof(UnsignedBits) * 8) - 1); + static const UnsignedBits MIN_KEY = UnsignedBits(-1); + static const UnsignedBits MAX_KEY = UnsignedBits(-1) ^ HIGH_BIT; + + static __device__ __forceinline__ UnsignedBits TwiddleIn(UnsignedBits key) + { + UnsignedBits mask = (key & HIGH_BIT) ? UnsignedBits(-1) : HIGH_BIT; + return key ^ mask; + }; + + static __device__ __forceinline__ UnsignedBits TwiddleOut(UnsignedBits key) + { + UnsignedBits mask = (key & HIGH_BIT) ? HIGH_BIT : UnsignedBits(-1); + return key ^ mask; + }; + + enum + { + PRIMITIVE = true, + NULL_TYPE = false, + }; +}; + +#endif // DOXYGEN_SHOULD_SKIP_THIS + + +/** + * \brief Numeric type traits + */ +template struct NumericTraits : BaseTraits {}; + +#ifndef DOXYGEN_SHOULD_SKIP_THIS // Do not document + +template <> struct NumericTraits : BaseTraits {}; + +template <> struct NumericTraits : BaseTraits<(std::numeric_limits::is_signed) ? SIGNED_INTEGER : UNSIGNED_INTEGER, true, false, unsigned char> {}; +template <> struct NumericTraits : BaseTraits {}; +template <> struct NumericTraits : BaseTraits {}; +template <> struct NumericTraits : BaseTraits {}; +template <> struct NumericTraits : BaseTraits {}; +template <> struct NumericTraits : BaseTraits {}; + +template <> struct NumericTraits : BaseTraits {}; +template <> struct NumericTraits : BaseTraits {}; +template <> struct NumericTraits : BaseTraits {}; +template <> struct NumericTraits : BaseTraits {}; +template <> struct NumericTraits : BaseTraits {}; + +template <> struct NumericTraits : BaseTraits {}; +template <> struct NumericTraits : BaseTraits {}; + +#endif // DOXYGEN_SHOULD_SKIP_THIS + + +/** + * \brief Type traits + */ +template +struct Traits : NumericTraits::Type> {}; + + + +/** @} */ // end group UtilModule + +} // CUB namespace +CUB_NS_POSTFIX // Optional outer namespace(s) diff --git a/lib/kokkos/TPL/cub/util_vector.cuh b/lib/kokkos/TPL/cub/util_vector.cuh new file mode 100755 index 0000000000..9a432dc582 --- /dev/null +++ b/lib/kokkos/TPL/cub/util_vector.cuh @@ -0,0 +1,166 @@ +/****************************************************************************** + * Copyright (c) 2011, Duane Merrill. All rights reserved. + * Copyright (c) 2011-2013, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the NVIDIA CORPORATION nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + ******************************************************************************/ + +/** + * \file + * Vector type inference utilities + */ + +#pragma once + +#include + +#include "util_namespace.cuh" + +/// Optional outer namespace(s) +CUB_NS_PREFIX + +/// CUB namespace +namespace cub { + + +/** + * \addtogroup UtilModule + * @{ + */ + + +/****************************************************************************** + * Vector type inference utilities. For example: + * + * typename VectorHelper::Type // Aliases uint2 + * + ******************************************************************************/ + +/** + * \brief Exposes a member typedef \p Type that names the corresponding CUDA vector type if one exists. Otherwise \p Type refers to the VectorHelper structure itself, which will wrap the corresponding \p x, \p y, etc. vector fields. + */ +template struct VectorHelper; + +#ifndef DOXYGEN_SHOULD_SKIP_THIS // Do not document + +enum +{ + /// The maximum number of elements in CUDA vector types + MAX_VEC_ELEMENTS = 4, +}; + + +/** + * Generic vector-1 type + */ +template +struct VectorHelper +{ + enum { BUILT_IN = false }; + + T x; + + typedef VectorHelper Type; +}; + +/** + * Generic vector-2 type + */ +template +struct VectorHelper +{ + enum { BUILT_IN = false }; + + T x; + T y; + + typedef VectorHelper Type; +}; + +/** + * Generic vector-3 type + */ +template +struct VectorHelper +{ + enum { BUILT_IN = false }; + + T x; + T y; + T z; + + typedef VectorHelper Type; +}; + +/** + * Generic vector-4 type + */ +template +struct VectorHelper +{ + enum { BUILT_IN = false }; + + T x; + T y; + T z; + T w; + + typedef VectorHelper Type; +}; + +/** + * Macro for expanding partially-specialized built-in vector types + */ +#define CUB_DEFINE_VECTOR_TYPE(base_type,short_type) \ + template<> struct VectorHelper { typedef short_type##1 Type; enum { BUILT_IN = true }; }; \ + template<> struct VectorHelper { typedef short_type##2 Type; enum { BUILT_IN = true }; }; \ + template<> struct VectorHelper { typedef short_type##3 Type; enum { BUILT_IN = true }; }; \ + template<> struct VectorHelper { typedef short_type##4 Type; enum { BUILT_IN = true }; }; + +// Expand CUDA vector types for built-in primitives +CUB_DEFINE_VECTOR_TYPE(char, char) +CUB_DEFINE_VECTOR_TYPE(signed char, char) +CUB_DEFINE_VECTOR_TYPE(short, short) +CUB_DEFINE_VECTOR_TYPE(int, int) +CUB_DEFINE_VECTOR_TYPE(long, long) +CUB_DEFINE_VECTOR_TYPE(long long, longlong) +CUB_DEFINE_VECTOR_TYPE(unsigned char, uchar) +CUB_DEFINE_VECTOR_TYPE(unsigned short, ushort) +CUB_DEFINE_VECTOR_TYPE(unsigned int, uint) +CUB_DEFINE_VECTOR_TYPE(unsigned long, ulong) +CUB_DEFINE_VECTOR_TYPE(unsigned long long, ulonglong) +CUB_DEFINE_VECTOR_TYPE(float, float) +CUB_DEFINE_VECTOR_TYPE(double, double) +CUB_DEFINE_VECTOR_TYPE(bool, uchar) + +// Undefine macros +#undef CUB_DEFINE_VECTOR_TYPE + +#endif // DOXYGEN_SHOULD_SKIP_THIS + + +/** @} */ // end group UtilModule + +} // CUB namespace +CUB_NS_POSTFIX // Optional outer namespace(s) diff --git a/lib/kokkos/TPL/cub/warp/specializations/warp_reduce_shfl.cuh b/lib/kokkos/TPL/cub/warp/specializations/warp_reduce_shfl.cuh new file mode 100755 index 0000000000..317b629900 --- /dev/null +++ b/lib/kokkos/TPL/cub/warp/specializations/warp_reduce_shfl.cuh @@ -0,0 +1,358 @@ +/****************************************************************************** + * Copyright (c) 2011, Duane Merrill. All rights reserved. + * Copyright (c) 2011-2013, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the NVIDIA CORPORATION nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + ******************************************************************************/ + +/** + * \file + * cub::WarpReduceShfl provides SHFL-based variants of parallel reduction across CUDA warps. + */ + +#pragma once + +#include "../../thread/thread_operators.cuh" +#include "../../util_ptx.cuh" +#include "../../util_type.cuh" +#include "../../util_macro.cuh" +#include "../../util_namespace.cuh" + +/// Optional outer namespace(s) +CUB_NS_PREFIX + +/// CUB namespace +namespace cub { + + +/** + * \brief WarpReduceShfl provides SHFL-based variants of parallel reduction across CUDA warps. + */ +template < + typename T, ///< Data type being reduced + int LOGICAL_WARPS, ///< Number of logical warps entrant + int LOGICAL_WARP_THREADS> ///< Number of threads per logical warp +struct WarpReduceShfl +{ + /****************************************************************************** + * Constants and typedefs + ******************************************************************************/ + + enum + { + /// The number of warp reduction steps + STEPS = Log2::VALUE, + + // The 5-bit SHFL mask for logically splitting warps into sub-segments + SHFL_MASK = (-1 << STEPS) & 31, + + // The 5-bit SFHL clamp + SHFL_CLAMP = LOGICAL_WARP_THREADS - 1, + + // The packed C argument (mask starts 8 bits up) + SHFL_C = (SHFL_MASK << 8) | SHFL_CLAMP, + }; + + + /// Shared memory storage layout type + typedef NullType TempStorage; + + + /****************************************************************************** + * Thread fields + ******************************************************************************/ + + int warp_id; + int lane_id; + + + /****************************************************************************** + * Construction + ******************************************************************************/ + + /// Constructor + __device__ __forceinline__ WarpReduceShfl( + TempStorage &temp_storage, + int warp_id, + int lane_id) + : + warp_id(warp_id), + lane_id(lane_id) + {} + + + /****************************************************************************** + * Operation + ******************************************************************************/ + + /// Summation (single-SHFL) + template < + bool FULL_WARPS, ///< Whether all lanes in each warp are contributing a valid fold of items + int FOLDED_ITEMS_PER_LANE> ///< Number of items folded into each lane + __device__ __forceinline__ T Sum( + T input, ///< [in] Calling thread's input + int folded_items_per_warp, ///< [in] Total number of valid items folded into each logical warp + Int2Type single_shfl) ///< [in] Marker type indicating whether only one SHFL instruction is required + { + unsigned int output = reinterpret_cast(input); + + // Iterate reduction steps + #pragma unroll + for (int STEP = 0; STEP < STEPS; STEP++) + { + const int OFFSET = 1 << STEP; + + if (FULL_WARPS) + { + // Use predicate set from SHFL to guard against invalid peers + asm( + "{" + " .reg .u32 r0;" + " .reg .pred p;" + " shfl.down.b32 r0|p, %1, %2, %3;" + " @p add.u32 r0, r0, %4;" + " mov.u32 %0, r0;" + "}" + : "=r"(output) : "r"(output), "r"(OFFSET), "r"(SHFL_C), "r"(output)); + } + else + { + // Set range predicate to guard against invalid peers + asm( + "{" + " .reg .u32 r0;" + " .reg .pred p;" + " shfl.down.b32 r0, %1, %2, %3;" + " setp.lt.u32 p, %5, %6;" + " mov.u32 %0, %1;" + " @p add.u32 %0, %1, r0;" + "}" + : "=r"(output) : "r"(output), "r"(OFFSET), "r"(SHFL_C), "r"(output), "r"((lane_id + OFFSET) * FOLDED_ITEMS_PER_LANE), "r"(folded_items_per_warp)); + } + } + + return output; + } + + + /// Summation (multi-SHFL) + template < + bool FULL_WARPS, ///< Whether all lanes in each warp are contributing a valid fold of items + int FOLDED_ITEMS_PER_LANE> ///< Number of items folded into each lane + __device__ __forceinline__ T Sum( + T input, ///< [in] Calling thread's input + int folded_items_per_warp, ///< [in] Total number of valid items folded into each logical warp + Int2Type single_shfl) ///< [in] Marker type indicating whether only one SHFL instruction is required + { + // Delegate to generic reduce + return Reduce(input, folded_items_per_warp, cub::Sum()); + } + + + /// Summation (float) + template < + bool FULL_WARPS, ///< Whether all lanes in each warp are contributing a valid fold of items + int FOLDED_ITEMS_PER_LANE> ///< Number of items folded into each lane + __device__ __forceinline__ float Sum( + float input, ///< [in] Calling thread's input + int folded_items_per_warp) ///< [in] Total number of valid items folded into each logical warp + { + T output = input; + + // Iterate reduction steps + #pragma unroll + for (int STEP = 0; STEP < STEPS; STEP++) + { + const int OFFSET = 1 << STEP; + + if (FULL_WARPS) + { + // Use predicate set from SHFL to guard against invalid peers + asm( + "{" + " .reg .f32 r0;" + " .reg .pred p;" + " shfl.down.b32 r0|p, %1, %2, %3;" + " @p add.f32 r0, r0, %4;" + " mov.f32 %0, r0;" + "}" + : "=f"(output) : "f"(output), "r"(OFFSET), "r"(SHFL_C), "f"(output)); + } + else + { + // Set range predicate to guard against invalid peers + asm( + "{" + " .reg .f32 r0;" + " .reg .pred p;" + " shfl.down.b32 r0, %1, %2, %3;" + " setp.lt.u32 p, %5, %6;" + " mov.f32 %0, %1;" + " @p add.f32 %0, %0, r0;" + "}" + : "=f"(output) : "f"(output), "r"(OFFSET), "r"(SHFL_C), "f"(output), "r"((lane_id + OFFSET) * FOLDED_ITEMS_PER_LANE), "r"(folded_items_per_warp)); + } + } + + return output; + } + + /// Summation (generic) + template < + bool FULL_WARPS, ///< Whether all lanes in each warp are contributing a valid fold of items + int FOLDED_ITEMS_PER_LANE, ///< Number of items folded into each lane + typename _T> + __device__ __forceinline__ _T Sum( + _T input, ///< [in] Calling thread's input + int folded_items_per_warp) ///< [in] Total number of valid items folded into each logical warp + { + // Whether sharing can be done with a single SHFL instruction (vs multiple SFHL instructions) + Int2Type<(Traits<_T>::PRIMITIVE) && (sizeof(_T) <= sizeof(unsigned int))> single_shfl; + + return Sum(input, folded_items_per_warp, single_shfl); + } + + + /// Reduction + template < + bool FULL_WARPS, ///< Whether all lanes in each warp are contributing a valid fold of items + int FOLDED_ITEMS_PER_LANE, ///< Number of items folded into each lane + typename ReductionOp> + __device__ __forceinline__ T Reduce( + T input, ///< [in] Calling thread's input + int folded_items_per_warp, ///< [in] Total number of valid items folded into each logical warp + ReductionOp reduction_op) ///< [in] Binary reduction operator + { + typedef typename WordAlignment::ShuffleWord ShuffleWord; + + const int WORDS = (sizeof(T) + sizeof(ShuffleWord) - 1) / sizeof(ShuffleWord); + T output = input; + T temp; + ShuffleWord *temp_alias = reinterpret_cast(&temp); + ShuffleWord *output_alias = reinterpret_cast(&output); + + // Iterate scan steps + #pragma unroll + for (int STEP = 0; STEP < STEPS; STEP++) + { + // Grab addend from peer + const int OFFSET = 1 << STEP; + + #pragma unroll + for (int WORD = 0; WORD < WORDS; ++WORD) + { + unsigned int shuffle_word = output_alias[WORD]; + asm( + " shfl.down.b32 %0, %1, %2, %3;" + : "=r"(shuffle_word) : "r"(shuffle_word), "r"(OFFSET), "r"(SHFL_C)); + temp_alias[WORD] = (ShuffleWord) shuffle_word; + } + + // Perform reduction op if from a valid peer + if (FULL_WARPS) + { + if (lane_id < LOGICAL_WARP_THREADS - OFFSET) + output = reduction_op(output, temp); + } + else + { + if (((lane_id + OFFSET) * FOLDED_ITEMS_PER_LANE) < folded_items_per_warp) + output = reduction_op(output, temp); + } + } + + return output; + } + + + /// Segmented reduction + template < + bool HEAD_SEGMENTED, ///< Whether flags indicate a segment-head or a segment-tail + typename Flag, + typename ReductionOp> + __device__ __forceinline__ T SegmentedReduce( + T input, ///< [in] Calling thread's input + Flag flag, ///< [in] Whether or not the current lane is a segment head/tail + ReductionOp reduction_op) ///< [in] Binary reduction operator + { + typedef typename WordAlignment::ShuffleWord ShuffleWord; + + T output = input; + + const int WORDS = (sizeof(T) + sizeof(ShuffleWord) - 1) / sizeof(ShuffleWord); + T temp; + ShuffleWord *temp_alias = reinterpret_cast(&temp); + ShuffleWord *output_alias = reinterpret_cast(&output); + + // Get the start flags for each thread in the warp. + int warp_flags = __ballot(flag); + + if (!HEAD_SEGMENTED) + warp_flags <<= 1; + + // Keep bits above the current thread. + warp_flags &= LaneMaskGt(); + + // Accommodate packing of multiple logical warps in a single physical warp + if ((LOGICAL_WARPS > 1) && (LOGICAL_WARP_THREADS < 32)) + warp_flags >>= (warp_id * LOGICAL_WARP_THREADS); + + // Find next flag + int next_flag = __clz(__brev(warp_flags)); + + // Clip the next segment at the warp boundary if necessary + if (LOGICAL_WARP_THREADS != 32) + next_flag = CUB_MIN(next_flag, LOGICAL_WARP_THREADS); + + // Iterate scan steps + #pragma unroll + for (int STEP = 0; STEP < STEPS; STEP++) + { + const int OFFSET = 1 << STEP; + + // Grab addend from peer + #pragma unroll + for (int WORD = 0; WORD < WORDS; ++WORD) + { + unsigned int shuffle_word = output_alias[WORD]; + + asm( + " shfl.down.b32 %0, %1, %2, %3;" + : "=r"(shuffle_word) : "r"(shuffle_word), "r"(OFFSET), "r"(SHFL_C)); + temp_alias[WORD] = (ShuffleWord) shuffle_word; + + } + + // Perform reduction op if valid + if (OFFSET < next_flag - lane_id) + output = reduction_op(output, temp); + } + + return output; + } +}; + + +} // CUB namespace +CUB_NS_POSTFIX // Optional outer namespace(s) diff --git a/lib/kokkos/TPL/cub/warp/specializations/warp_reduce_smem.cuh b/lib/kokkos/TPL/cub/warp/specializations/warp_reduce_smem.cuh new file mode 100755 index 0000000000..a32d5fdd74 --- /dev/null +++ b/lib/kokkos/TPL/cub/warp/specializations/warp_reduce_smem.cuh @@ -0,0 +1,291 @@ +/****************************************************************************** + * Copyright (c) 2011, Duane Merrill. All rights reserved. + * Copyright (c) 2011-2013, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the NVIDIA CORPORATION nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + ******************************************************************************/ + +/** + * \file + * cub::WarpReduceSmem provides smem-based variants of parallel reduction across CUDA warps. + */ + +#pragma once + +#include "../../thread/thread_operators.cuh" +#include "../../thread/thread_load.cuh" +#include "../../thread/thread_store.cuh" +#include "../../util_type.cuh" +#include "../../util_namespace.cuh" + +/// Optional outer namespace(s) +CUB_NS_PREFIX + +/// CUB namespace +namespace cub { + +/** + * \brief WarpReduceSmem provides smem-based variants of parallel reduction across CUDA warps. + */ +template < + typename T, ///< Data type being reduced + int LOGICAL_WARPS, ///< Number of logical warps entrant + int LOGICAL_WARP_THREADS> ///< Number of threads per logical warp +struct WarpReduceSmem +{ + /****************************************************************************** + * Constants and typedefs + ******************************************************************************/ + + enum + { + /// Whether the logical warp size is a power-of-two + POW_OF_TWO = ((LOGICAL_WARP_THREADS & (LOGICAL_WARP_THREADS - 1)) == 0), + + /// The number of warp scan steps + STEPS = Log2::VALUE, + + /// The number of threads in half a warp + HALF_WARP_THREADS = 1 << (STEPS - 1), + + /// The number of shared memory elements per warp + WARP_SMEM_ELEMENTS = LOGICAL_WARP_THREADS + HALF_WARP_THREADS, + }; + + /// Shared memory flag type + typedef unsigned char SmemFlag; + + /// Shared memory storage layout type (1.5 warps-worth of elements for each warp) + typedef T _TempStorage[LOGICAL_WARPS][WARP_SMEM_ELEMENTS]; + + // Alias wrapper allowing storage to be unioned + struct TempStorage : Uninitialized<_TempStorage> {}; + + + /****************************************************************************** + * Thread fields + ******************************************************************************/ + + _TempStorage &temp_storage; + int warp_id; + int lane_id; + + + /****************************************************************************** + * Construction + ******************************************************************************/ + + /// Constructor + __device__ __forceinline__ WarpReduceSmem( + TempStorage &temp_storage, + int warp_id, + int lane_id) + : + temp_storage(temp_storage.Alias()), + warp_id(warp_id), + lane_id(lane_id) + {} + + + /****************************************************************************** + * Operation + ******************************************************************************/ + + /** + * Reduction + */ + template < + bool FULL_WARPS, ///< Whether all lanes in each warp are contributing a valid fold of items + int FOLDED_ITEMS_PER_LANE, ///< Number of items folded into each lane + typename ReductionOp> + __device__ __forceinline__ T Reduce( + T input, ///< [in] Calling thread's input + int folded_items_per_warp, ///< [in] Total number of valid items folded into each logical warp + ReductionOp reduction_op) ///< [in] Reduction operator + { + for (int STEP = 0; STEP < STEPS; STEP++) + { + const int OFFSET = 1 << STEP; + + // Share input through buffer + ThreadStore(&temp_storage[warp_id][lane_id], input); + + // Update input if peer_addend is in range + if ((FULL_WARPS && POW_OF_TWO) || ((lane_id + OFFSET) * FOLDED_ITEMS_PER_LANE < folded_items_per_warp)) + { + T peer_addend = ThreadLoad(&temp_storage[warp_id][lane_id + OFFSET]); + input = reduction_op(input, peer_addend); + } + } + + return input; + } + + + /** + * Segmented reduction + */ + template < + bool HEAD_SEGMENTED, ///< Whether flags indicate a segment-head or a segment-tail + typename Flag, + typename ReductionOp> + __device__ __forceinline__ T SegmentedReduce( + T input, ///< [in] Calling thread's input + Flag flag, ///< [in] Whether or not the current lane is a segment head/tail + ReductionOp reduction_op) ///< [in] Reduction operator + { + #if CUB_PTX_ARCH >= 200 + + // Ballot-based segmented reduce + + // Get the start flags for each thread in the warp. + int warp_flags = __ballot(flag); + + if (!HEAD_SEGMENTED) + warp_flags <<= 1; + + // Keep bits above the current thread. + warp_flags &= LaneMaskGt(); + + // Accommodate packing of multiple logical warps in a single physical warp + if ((LOGICAL_WARPS > 1) && (LOGICAL_WARP_THREADS < 32)) + warp_flags >>= (warp_id * LOGICAL_WARP_THREADS); + + // Find next flag + int next_flag = __clz(__brev(warp_flags)); + + // Clip the next segment at the warp boundary if necessary + if (LOGICAL_WARP_THREADS != 32) + next_flag = CUB_MIN(next_flag, LOGICAL_WARP_THREADS); + + for (int STEP = 0; STEP < STEPS; STEP++) + { + const int OFFSET = 1 << STEP; + + // Share input into buffer + ThreadStore(&temp_storage[warp_id][lane_id], input); + + // Update input if peer_addend is in range + if (OFFSET < next_flag - lane_id) + { + T peer_addend = ThreadLoad(&temp_storage[warp_id][lane_id + OFFSET]); + input = reduction_op(input, peer_addend); + } + } + + return input; + + #else + + // Smem-based segmented reduce + + enum + { + UNSET = 0x0, // Is initially unset + SET = 0x1, // Is initially set + SEEN = 0x2, // Has seen another head flag from a successor peer + }; + + // Alias flags onto shared data storage + volatile SmemFlag *flag_storage = reinterpret_cast(temp_storage[warp_id]); + + SmemFlag flag_status = (flag) ? SET : UNSET; + + for (int STEP = 0; STEP < STEPS; STEP++) + { + const int OFFSET = 1 << STEP; + + // Share input through buffer + ThreadStore(&temp_storage[warp_id][lane_id], input); + + // Get peer from buffer + T peer_addend = ThreadLoad(&temp_storage[warp_id][lane_id + OFFSET]); + + // Share flag through buffer + flag_storage[lane_id] = flag_status; + + // Get peer flag from buffer + SmemFlag peer_flag_status = flag_storage[lane_id + OFFSET]; + + // Update input if peer was in range + if (lane_id < LOGICAL_WARP_THREADS - OFFSET) + { + if (HEAD_SEGMENTED) + { + // Head-segmented + if ((flag_status & SEEN) == 0) + { + // Has not seen a more distant head flag + if (peer_flag_status & SET) + { + // Has now seen a head flag + flag_status |= SEEN; + } + else + { + // Peer is not a head flag: grab its count + input = reduction_op(input, peer_addend); + } + + // Update seen status to include that of peer + flag_status |= (peer_flag_status & SEEN); + } + } + else + { + // Tail-segmented. Simply propagate flag status + if (!flag_status) + { + input = reduction_op(input, peer_addend); + flag_status |= peer_flag_status; + } + + } + } + } + + return input; + + #endif + } + + + /** + * Summation + */ + template < + bool FULL_WARPS, ///< Whether all lanes in each warp are contributing a valid fold of items + int FOLDED_ITEMS_PER_LANE> ///< Number of items folded into each lane + __device__ __forceinline__ T Sum( + T input, ///< [in] Calling thread's input + int folded_items_per_warp) ///< [in] Total number of valid items folded into each logical warp + { + return Reduce(input, folded_items_per_warp, cub::Sum()); + } + +}; + + +} // CUB namespace +CUB_NS_POSTFIX // Optional outer namespace(s) diff --git a/lib/kokkos/TPL/cub/warp/specializations/warp_scan_shfl.cuh b/lib/kokkos/TPL/cub/warp/specializations/warp_scan_shfl.cuh new file mode 100755 index 0000000000..5585396cec --- /dev/null +++ b/lib/kokkos/TPL/cub/warp/specializations/warp_scan_shfl.cuh @@ -0,0 +1,371 @@ +/****************************************************************************** + * Copyright (c) 2011, Duane Merrill. All rights reserved. + * Copyright (c) 2011-2013, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the NVIDIA CORPORATION nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + ******************************************************************************/ + +/** + * \file + * cub::WarpScanShfl provides SHFL-based variants of parallel prefix scan across CUDA warps. + */ + +#pragma once + +#include "../../thread/thread_operators.cuh" +#include "../../util_type.cuh" +#include "../../util_ptx.cuh" +#include "../../util_namespace.cuh" + +/// Optional outer namespace(s) +CUB_NS_PREFIX + +/// CUB namespace +namespace cub { + +/** + * \brief WarpScanShfl provides SHFL-based variants of parallel prefix scan across CUDA warps. + */ +template < + typename T, ///< Data type being scanned + int LOGICAL_WARPS, ///< Number of logical warps entrant + int LOGICAL_WARP_THREADS> ///< Number of threads per logical warp +struct WarpScanShfl +{ + + /****************************************************************************** + * Constants and typedefs + ******************************************************************************/ + + enum + { + /// The number of warp scan steps + STEPS = Log2::VALUE, + + // The 5-bit SHFL mask for logically splitting warps into sub-segments starts 8-bits up + SHFL_C = ((-1 << STEPS) & 31) << 8, + }; + + /// Shared memory storage layout type + typedef NullType TempStorage; + + + /****************************************************************************** + * Thread fields + ******************************************************************************/ + + int warp_id; + int lane_id; + + /****************************************************************************** + * Construction + ******************************************************************************/ + + /// Constructor + __device__ __forceinline__ WarpScanShfl( + TempStorage &temp_storage, + int warp_id, + int lane_id) + : + warp_id(warp_id), + lane_id(lane_id) + {} + + + /****************************************************************************** + * Operation + ******************************************************************************/ + + /// Broadcast + __device__ __forceinline__ T Broadcast( + T input, ///< [in] The value to broadcast + int src_lane) ///< [in] Which warp lane is to do the broadcasting + { + typedef typename WordAlignment::ShuffleWord ShuffleWord; + + const int WORDS = (sizeof(T) + sizeof(ShuffleWord) - 1) / sizeof(ShuffleWord); + T output; + ShuffleWord *output_alias = reinterpret_cast(&output); + ShuffleWord *input_alias = reinterpret_cast(&input); + + #pragma unroll + for (int WORD = 0; WORD < WORDS; ++WORD) + { + unsigned int shuffle_word = input_alias[WORD]; + asm("shfl.idx.b32 %0, %1, %2, %3;" + : "=r"(shuffle_word) : "r"(shuffle_word), "r"(src_lane), "r"(LOGICAL_WARP_THREADS - 1)); + output_alias[WORD] = (ShuffleWord) shuffle_word; + } + + return output; + } + + + //--------------------------------------------------------------------- + // Inclusive operations + //--------------------------------------------------------------------- + + /// Inclusive prefix sum with aggregate (single-SHFL) + __device__ __forceinline__ void InclusiveSum( + T input, ///< [in] Calling thread's input item. + T &output, ///< [out] Calling thread's output item. May be aliased with \p input. + T &warp_aggregate, ///< [out] Warp-wide aggregate reduction of input items. + Int2Type single_shfl) + { + unsigned int temp = reinterpret_cast(input); + + // Iterate scan steps + #pragma unroll + for (int STEP = 0; STEP < STEPS; STEP++) + { + // Use predicate set from SHFL to guard against invalid peers + asm( + "{" + " .reg .u32 r0;" + " .reg .pred p;" + " shfl.up.b32 r0|p, %1, %2, %3;" + " @p add.u32 r0, r0, %4;" + " mov.u32 %0, r0;" + "}" + : "=r"(temp) : "r"(temp), "r"(1 << STEP), "r"(SHFL_C), "r"(temp)); + } + + output = temp; + + // Grab aggregate from last warp lane + warp_aggregate = Broadcast(output, LOGICAL_WARP_THREADS - 1); + } + + + /// Inclusive prefix sum with aggregate (multi-SHFL) + __device__ __forceinline__ void InclusiveSum( + T input, ///< [in] Calling thread's input item. + T &output, ///< [out] Calling thread's output item. May be aliased with \p input. + T &warp_aggregate, ///< [out] Warp-wide aggregate reduction of input items. + Int2Type single_shfl) ///< [in] Marker type indicating whether only one SHFL instruction is required + { + // Delegate to generic scan + InclusiveScan(input, output, Sum(), warp_aggregate); + } + + + /// Inclusive prefix sum with aggregate (specialized for float) + __device__ __forceinline__ void InclusiveSum( + float input, ///< [in] Calling thread's input item. + float &output, ///< [out] Calling thread's output item. May be aliased with \p input. + float &warp_aggregate) ///< [out] Warp-wide aggregate reduction of input items. + { + output = input; + + // Iterate scan steps + #pragma unroll + for (int STEP = 0; STEP < STEPS; STEP++) + { + // Use predicate set from SHFL to guard against invalid peers + asm( + "{" + " .reg .f32 r0;" + " .reg .pred p;" + " shfl.up.b32 r0|p, %1, %2, %3;" + " @p add.f32 r0, r0, %4;" + " mov.f32 %0, r0;" + "}" + : "=f"(output) : "f"(output), "r"(1 << STEP), "r"(SHFL_C), "f"(output)); + } + + // Grab aggregate from last warp lane + warp_aggregate = Broadcast(output, LOGICAL_WARP_THREADS - 1); + } + + + /// Inclusive prefix sum with aggregate (specialized for unsigned long long) + __device__ __forceinline__ void InclusiveSum( + unsigned long long input, ///< [in] Calling thread's input item. + unsigned long long &output, ///< [out] Calling thread's output item. May be aliased with \p input. + unsigned long long &warp_aggregate) ///< [out] Warp-wide aggregate reduction of input items. + { + output = input; + + // Iterate scan steps + #pragma unroll + for (int STEP = 0; STEP < STEPS; STEP++) + { + // Use predicate set from SHFL to guard against invalid peers + asm( + "{" + " .reg .u32 r0;" + " .reg .u32 r1;" + " .reg .u32 lo;" + " .reg .u32 hi;" + " .reg .pred p;" + " mov.b64 {lo, hi}, %1;" + " shfl.up.b32 r0|p, lo, %2, %3;" + " shfl.up.b32 r1|p, hi, %2, %3;" + " @p add.cc.u32 r0, r0, lo;" + " @p addc.u32 r1, r1, hi;" + " mov.b64 %0, {r0, r1};" + "}" + : "=l"(output) : "l"(output), "r"(1 << STEP), "r"(SHFL_C)); + } + + // Grab aggregate from last warp lane + warp_aggregate = Broadcast(output, LOGICAL_WARP_THREADS - 1); + } + + + /// Inclusive prefix sum with aggregate (generic) + template + __device__ __forceinline__ void InclusiveSum( + _T input, ///< [in] Calling thread's input item. + _T &output, ///< [out] Calling thread's output item. May be aliased with \p input. + _T &warp_aggregate) ///< [out] Warp-wide aggregate reduction of input items. + { + // Whether sharing can be done with a single SHFL instruction (vs multiple SFHL instructions) + Int2Type<(Traits<_T>::PRIMITIVE) && (sizeof(_T) <= sizeof(unsigned int))> single_shfl; + + InclusiveSum(input, output, warp_aggregate, single_shfl); + } + + + /// Inclusive prefix sum + __device__ __forceinline__ void InclusiveSum( + T input, ///< [in] Calling thread's input item. + T &output) ///< [out] Calling thread's output item. May be aliased with \p input. + { + T warp_aggregate; + InclusiveSum(input, output, warp_aggregate); + } + + + /// Inclusive scan with aggregate + template + __device__ __forceinline__ void InclusiveScan( + T input, ///< [in] Calling thread's input item. + T &output, ///< [out] Calling thread's output item. May be aliased with \p input. + ScanOp scan_op, ///< [in] Binary scan operator + T &warp_aggregate) ///< [out] Warp-wide aggregate reduction of input items. + { + output = input; + + // Iterate scan steps + #pragma unroll + for (int STEP = 0; STEP < STEPS; STEP++) + { + // Grab addend from peer + const int OFFSET = 1 << STEP; + T temp = ShuffleUp(output, OFFSET); + + // Perform scan op if from a valid peer + if (lane_id >= OFFSET) + output = scan_op(temp, output); + } + + // Grab aggregate from last warp lane + warp_aggregate = Broadcast(output, LOGICAL_WARP_THREADS - 1); + } + + + /// Inclusive scan + template + __device__ __forceinline__ void InclusiveScan( + T input, ///< [in] Calling thread's input item. + T &output, ///< [out] Calling thread's output item. May be aliased with \p input. + ScanOp scan_op) ///< [in] Binary scan operator + { + T warp_aggregate; + InclusiveScan(input, output, scan_op, warp_aggregate); + } + + + //--------------------------------------------------------------------- + // Exclusive operations + //--------------------------------------------------------------------- + + /// Exclusive scan with aggregate + template + __device__ __forceinline__ void ExclusiveScan( + T input, ///< [in] Calling thread's input item. + T &output, ///< [out] Calling thread's output item. May be aliased with \p input. + T identity, ///< [in] Identity value + ScanOp scan_op, ///< [in] Binary scan operator + T &warp_aggregate) ///< [out] Warp-wide aggregate reduction of input items. + { + // Compute inclusive scan + T inclusive; + InclusiveScan(input, inclusive, scan_op, warp_aggregate); + + // Grab result from predecessor + T exclusive = ShuffleUp(inclusive, 1); + + output = (lane_id == 0) ? + identity : + exclusive; + } + + + /// Exclusive scan + template + __device__ __forceinline__ void ExclusiveScan( + T input, ///< [in] Calling thread's input item. + T &output, ///< [out] Calling thread's output item. May be aliased with \p input. + T identity, ///< [in] Identity value + ScanOp scan_op) ///< [in] Binary scan operator + { + T warp_aggregate; + ExclusiveScan(input, output, identity, scan_op, warp_aggregate); + } + + + /// Exclusive scan with aggregate, without identity + template + __device__ __forceinline__ void ExclusiveScan( + T input, ///< [in] Calling thread's input item. + T &output, ///< [out] Calling thread's output item. May be aliased with \p input. + ScanOp scan_op, ///< [in] Binary scan operator + T &warp_aggregate) ///< [out] Warp-wide aggregate reduction of input items. + { + // Compute inclusive scan + T inclusive; + InclusiveScan(input, inclusive, scan_op, warp_aggregate); + + // Grab result from predecessor + output = ShuffleUp(inclusive, 1); + } + + + /// Exclusive scan without identity + template + __device__ __forceinline__ void ExclusiveScan( + T input, ///< [in] Calling thread's input item. + T &output, ///< [out] Calling thread's output item. May be aliased with \p input. + ScanOp scan_op) ///< [in] Binary scan operator + { + T warp_aggregate; + ExclusiveScan(input, output, scan_op, warp_aggregate); + } +}; + + +} // CUB namespace +CUB_NS_POSTFIX // Optional outer namespace(s) diff --git a/lib/kokkos/TPL/cub/warp/specializations/warp_scan_smem.cuh b/lib/kokkos/TPL/cub/warp/specializations/warp_scan_smem.cuh new file mode 100755 index 0000000000..513b35ceff --- /dev/null +++ b/lib/kokkos/TPL/cub/warp/specializations/warp_scan_smem.cuh @@ -0,0 +1,327 @@ +/****************************************************************************** + * Copyright (c) 2011, Duane Merrill. All rights reserved. + * Copyright (c) 2011-2013, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the NVIDIA CORPORATION nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + ******************************************************************************/ + +/** + * \file + * cub::WarpScanSmem provides smem-based variants of parallel prefix scan across CUDA warps. + */ + +#pragma once + +#include "../../thread/thread_operators.cuh" +#include "../../thread/thread_load.cuh" +#include "../../thread/thread_store.cuh" +#include "../../util_type.cuh" +#include "../../util_namespace.cuh" + +/// Optional outer namespace(s) +CUB_NS_PREFIX + +/// CUB namespace +namespace cub { + +/** + * \brief WarpScanSmem provides smem-based variants of parallel prefix scan across CUDA warps. + */ +template < + typename T, ///< Data type being scanned + int LOGICAL_WARPS, ///< Number of logical warps entrant + int LOGICAL_WARP_THREADS> ///< Number of threads per logical warp +struct WarpScanSmem +{ + /****************************************************************************** + * Constants and typedefs + ******************************************************************************/ + + enum + { + /// The number of warp scan steps + STEPS = Log2::VALUE, + + /// The number of threads in half a warp + HALF_WARP_THREADS = 1 << (STEPS - 1), + + /// The number of shared memory elements per warp + WARP_SMEM_ELEMENTS = LOGICAL_WARP_THREADS + HALF_WARP_THREADS, + }; + + + /// Shared memory storage layout type (1.5 warps-worth of elements for each warp) + typedef T _TempStorage[LOGICAL_WARPS][WARP_SMEM_ELEMENTS]; + + // Alias wrapper allowing storage to be unioned + struct TempStorage : Uninitialized<_TempStorage> {}; + + + /****************************************************************************** + * Thread fields + ******************************************************************************/ + + _TempStorage &temp_storage; + unsigned int warp_id; + unsigned int lane_id; + + + /****************************************************************************** + * Construction + ******************************************************************************/ + + /// Constructor + __device__ __forceinline__ WarpScanSmem( + TempStorage &temp_storage, + int warp_id, + int lane_id) + : + temp_storage(temp_storage.Alias()), + warp_id(warp_id), + lane_id(lane_id) + {} + + + /****************************************************************************** + * Operation + ******************************************************************************/ + + /// Initialize identity padding (specialized for operations that have identity) + __device__ __forceinline__ void InitIdentity(Int2Type has_identity) + { + T identity = T(); + ThreadStore(&temp_storage[warp_id][lane_id], identity); + } + + + /// Initialize identity padding (specialized for operations without identity) + __device__ __forceinline__ void InitIdentity(Int2Type has_identity) + {} + + + /// Basic inclusive scan iteration(template unrolled, base-case specialization) + template < + bool HAS_IDENTITY, + typename ScanOp> + __device__ __forceinline__ void ScanStep( + T &partial, + ScanOp scan_op, + Int2Type step) + {} + + + /// Basic inclusive scan iteration (template unrolled, inductive-case specialization) + template < + bool HAS_IDENTITY, + int STEP, + typename ScanOp> + __device__ __forceinline__ void ScanStep( + T &partial, + ScanOp scan_op, + Int2Type step) + { + const int OFFSET = 1 << STEP; + + // Share partial into buffer + ThreadStore(&temp_storage[warp_id][HALF_WARP_THREADS + lane_id], partial); + + // Update partial if addend is in range + if (HAS_IDENTITY || (lane_id >= OFFSET)) + { + T addend = ThreadLoad(&temp_storage[warp_id][HALF_WARP_THREADS + lane_id - OFFSET]); + partial = scan_op(addend, partial); + } + + ScanStep(partial, scan_op, Int2Type()); + } + + + /// Broadcast + __device__ __forceinline__ T Broadcast( + T input, ///< [in] The value to broadcast + unsigned int src_lane) ///< [in] Which warp lane is to do the broadcasting + { + if (lane_id == src_lane) + { + ThreadStore(temp_storage[warp_id], input); + } + + return ThreadLoad(temp_storage[warp_id]); + } + + + /// Basic inclusive scan + template < + bool HAS_IDENTITY, + bool SHARE_FINAL, + typename ScanOp> + __device__ __forceinline__ T BasicScan( + T partial, ///< Calling thread's input partial reduction + ScanOp scan_op) ///< Binary associative scan functor + { + // Iterate scan steps + ScanStep(partial, scan_op, Int2Type<0>()); + + if (SHARE_FINAL) + { + // Share partial into buffer + ThreadStore(&temp_storage[warp_id][HALF_WARP_THREADS + lane_id], partial); + } + + return partial; + } + + + /// Inclusive prefix sum + __device__ __forceinline__ void InclusiveSum( + T input, ///< [in] Calling thread's input item. + T &output) ///< [out] Calling thread's output item. May be aliased with \p input. + { + const bool HAS_IDENTITY = Traits::PRIMITIVE; + + // Initialize identity region + InitIdentity(Int2Type()); + + // Compute inclusive warp scan (has identity, don't share final) + output = BasicScan(input, Sum()); + } + + + /// Inclusive prefix sum with aggregate + __device__ __forceinline__ void InclusiveSum( + T input, ///< [in] Calling thread's input item. + T &output, ///< [out] Calling thread's output item. May be aliased with \p input. + T &warp_aggregate) ///< [out] Warp-wide aggregate reduction of input items. + { + const bool HAS_IDENTITY = Traits::PRIMITIVE; + + // Initialize identity region + InitIdentity(Int2Type()); + + // Compute inclusive warp scan (has identity, share final) + output = BasicScan(input, Sum()); + + // Retrieve aggregate in warp-lane0 + warp_aggregate = ThreadLoad(&temp_storage[warp_id][WARP_SMEM_ELEMENTS - 1]); + } + + + /// Inclusive scan + template + __device__ __forceinline__ void InclusiveScan( + T input, ///< [in] Calling thread's input item. + T &output, ///< [out] Calling thread's output item. May be aliased with \p input. + ScanOp scan_op) ///< [in] Binary scan operator + { + // Compute inclusive warp scan (no identity, don't share final) + output = BasicScan(input, scan_op); + } + + + /// Inclusive scan with aggregate + template + __device__ __forceinline__ void InclusiveScan( + T input, ///< [in] Calling thread's input item. + T &output, ///< [out] Calling thread's output item. May be aliased with \p input. + ScanOp scan_op, ///< [in] Binary scan operator + T &warp_aggregate) ///< [out] Warp-wide aggregate reduction of input items. + { + // Compute inclusive warp scan (no identity, share final) + output = BasicScan(input, scan_op); + + // Retrieve aggregate + warp_aggregate = ThreadLoad(&temp_storage[warp_id][WARP_SMEM_ELEMENTS - 1]); + } + + /// Exclusive scan + template + __device__ __forceinline__ void ExclusiveScan( + T input, ///< [in] Calling thread's input item. + T &output, ///< [out] Calling thread's output item. May be aliased with \p input. + T identity, ///< [in] Identity value + ScanOp scan_op) ///< [in] Binary scan operator + { + // Initialize identity region + ThreadStore(&temp_storage[warp_id][lane_id], identity); + + // Compute inclusive warp scan (identity, share final) + T inclusive = BasicScan(input, scan_op); + + // Retrieve exclusive scan + output = ThreadLoad(&temp_storage[warp_id][HALF_WARP_THREADS + lane_id - 1]); + } + + + /// Exclusive scan with aggregate + template + __device__ __forceinline__ void ExclusiveScan( + T input, ///< [in] Calling thread's input item. + T &output, ///< [out] Calling thread's output item. May be aliased with \p input. + T identity, ///< [in] Identity value + ScanOp scan_op, ///< [in] Binary scan operator + T &warp_aggregate) ///< [out] Warp-wide aggregate reduction of input items. + { + // Exclusive warp scan (which does share final) + ExclusiveScan(input, output, identity, scan_op); + + // Retrieve aggregate + warp_aggregate = ThreadLoad(&temp_storage[warp_id][WARP_SMEM_ELEMENTS - 1]); + } + + + /// Exclusive scan without identity + template + __device__ __forceinline__ void ExclusiveScan( + T input, ///< [in] Calling thread's input item. + T &output, ///< [out] Calling thread's output item. May be aliased with \p input. + ScanOp scan_op) ///< [in] Binary scan operator + { + // Compute inclusive warp scan (no identity, share final) + T inclusive = BasicScan(input, scan_op); + + // Retrieve exclusive scan + output = ThreadLoad(&temp_storage[warp_id][HALF_WARP_THREADS + lane_id - 1]); + } + + + /// Exclusive scan with aggregate, without identity + template + __device__ __forceinline__ void ExclusiveScan( + T input, ///< [in] Calling thread's input item. + T &output, ///< [out] Calling thread's output item. May be aliased with \p input. + ScanOp scan_op, ///< [in] Binary scan operator + T &warp_aggregate) ///< [out] Warp-wide aggregate reduction of input items. + { + // Exclusive warp scan (which does share final) + ExclusiveScan(input, output, scan_op); + + // Retrieve aggregate + warp_aggregate = ThreadLoad(&temp_storage[warp_id][WARP_SMEM_ELEMENTS - 1]); + } + +}; + + +} // CUB namespace +CUB_NS_POSTFIX // Optional outer namespace(s) diff --git a/lib/kokkos/TPL/cub/warp/warp_reduce.cuh b/lib/kokkos/TPL/cub/warp/warp_reduce.cuh new file mode 100755 index 0000000000..548369da14 --- /dev/null +++ b/lib/kokkos/TPL/cub/warp/warp_reduce.cuh @@ -0,0 +1,677 @@ +/****************************************************************************** + * Copyright (c) 2011, Duane Merrill. All rights reserved. + * Copyright (c) 2011-2013, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the NVIDIA CORPORATION nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + ******************************************************************************/ + +/** + * \file + * The cub::WarpReduce class provides [collective](index.html#sec0) methods for computing a parallel reduction of items partitioned across CUDA warp threads. + */ + +#pragma once + +#include "specializations/warp_reduce_shfl.cuh" +#include "specializations/warp_reduce_smem.cuh" +#include "../thread/thread_operators.cuh" +#include "../util_arch.cuh" +#include "../util_type.cuh" +#include "../util_namespace.cuh" + +/// Optional outer namespace(s) +CUB_NS_PREFIX + +/// CUB namespace +namespace cub { + + +/** + * \addtogroup WarpModule + * @{ + */ + +/** + * \brief The WarpReduce class provides [collective](index.html#sec0) methods for computing a parallel reduction of items partitioned across CUDA warp threads. ![](warp_reduce_logo.png) + * + * \par Overview + * A reduction (or fold) + * uses a binary combining operator to compute a single aggregate from a list of input elements. + * + * \tparam T The reduction input/output element type + * \tparam LOGICAL_WARPS [optional] The number of entrant "logical" warps performing concurrent warp reductions. Default is 1. + * \tparam LOGICAL_WARP_THREADS [optional] The number of threads per "logical" warp (may be less than the number of hardware warp threads). Default is the warp size of the targeted CUDA compute-capability (e.g., 32 threads for SM20). + * + * \par Simple Examples + * \warpcollective{WarpReduce} + * \par + * The code snippet below illustrates four concurrent warp sum reductions within a block of + * 128 threads (one per each of the 32-thread warps). + * \par + * \code + * #include + * + * __global__ void ExampleKernel(...) + * { + * // Specialize WarpReduce for 4 warps on type int + * typedef cub::WarpReduce WarpReduce; + * + * // Allocate shared memory for WarpReduce + * __shared__ typename WarpReduce::TempStorage temp_storage; + * + * // Obtain one input item per thread + * int thread_data = ... + * + * // Return the warp-wide sums to each lane0 (threads 0, 32, 64, and 96) + * int aggregate = WarpReduce(temp_storage).Sum(thread_data); + * + * \endcode + * \par + * Suppose the set of input \p thread_data across the block of threads is 0, 1, 2, 3, ..., 127. + * The corresponding output \p aggregate in threads 0, 32, 64, and 96 will \p 496, \p 1520, + * \p 2544, and \p 3568, respectively (and is undefined in other threads). + * + * \par + * The code snippet below illustrates a single warp sum reduction within a block of + * 128 threads. + * \par + * \code + * #include + * + * __global__ void ExampleKernel(...) + * { + * // Specialize WarpReduce for one warp on type int + * typedef cub::WarpReduce WarpReduce; + * + * // Allocate shared memory for WarpReduce + * __shared__ typename WarpReduce::TempStorage temp_storage; + * ... + * + * // Only the first warp performs a reduction + * if (threadIdx.x < 32) + * { + * // Obtain one input item per thread + * int thread_data = ... + * + * // Return the warp-wide sum to lane0 + * int aggregate = WarpReduce(temp_storage).Sum(thread_data); + * + * \endcode + * \par + * Suppose the set of input \p thread_data across the warp of threads is 0, 1, 2, 3, ..., 31. + * The corresponding output \p aggregate in thread0 will be \p 496 (and is undefined in other threads). + * + * \par Usage and Performance Considerations + * - Supports "logical" warps smaller than the physical warp size (e.g., logical warps of 8 threads) + * - The number of entrant threads must be an multiple of \p LOGICAL_WARP_THREADS + * - Warp reductions are concurrent if more than one logical warp is participating + * - Uses special instructions when applicable (e.g., warp \p SHFL instructions) + * - Uses synchronization-free communication between warp lanes when applicable + * - Zero bank conflicts for most types + * - Computation is slightly more efficient (i.e., having lower instruction overhead) for: + * - Summation (vs. generic reduction) + * - The architecture's warp size is a whole multiple of \p LOGICAL_WARP_THREADS + * + */ +template < + typename T, + int LOGICAL_WARPS = 1, + int LOGICAL_WARP_THREADS = PtxArchProps::WARP_THREADS> +class WarpReduce +{ +private: + + /****************************************************************************** + * Constants and typedefs + ******************************************************************************/ + + enum + { + POW_OF_TWO = ((LOGICAL_WARP_THREADS & (LOGICAL_WARP_THREADS - 1)) == 0), + }; + +public: + + #ifndef DOXYGEN_SHOULD_SKIP_THIS // Do not document + + /// Internal specialization. Use SHFL-based reduction if (architecture is >= SM30) and ((only one logical warp) or (LOGICAL_WARP_THREADS is a power-of-two)) + typedef typename If<(CUB_PTX_ARCH >= 300) && ((LOGICAL_WARPS == 1) || POW_OF_TWO), + WarpReduceShfl, + WarpReduceSmem >::Type InternalWarpReduce; + + #endif // DOXYGEN_SHOULD_SKIP_THIS + + +private: + + /// Shared memory storage layout type for WarpReduce + typedef typename InternalWarpReduce::TempStorage _TempStorage; + + + /****************************************************************************** + * Thread fields + ******************************************************************************/ + + /// Shared storage reference + _TempStorage &temp_storage; + + /// Warp ID + int warp_id; + + /// Lane ID + int lane_id; + + + /****************************************************************************** + * Utility methods + ******************************************************************************/ + + /// Internal storage allocator + __device__ __forceinline__ _TempStorage& PrivateStorage() + { + __shared__ TempStorage private_storage; + return private_storage; + } + + +public: + + /// \smemstorage{WarpReduce} + struct TempStorage : Uninitialized<_TempStorage> {}; + + + /******************************************************************//** + * \name Collective constructors + *********************************************************************/ + //@{ + + + /** + * \brief Collective constructor for 1D thread blocks using a private static allocation of shared memory as temporary storage. Logical warp and lane identifiers are constructed from threadIdx.x. + * + */ + __device__ __forceinline__ WarpReduce() + : + temp_storage(PrivateStorage()), + warp_id((LOGICAL_WARPS == 1) ? + 0 : + threadIdx.x / LOGICAL_WARP_THREADS), + lane_id(((LOGICAL_WARPS == 1) || (LOGICAL_WARP_THREADS == PtxArchProps::WARP_THREADS)) ? + LaneId() : + threadIdx.x % LOGICAL_WARP_THREADS) + {} + + + /** + * \brief Collective constructor for 1D thread blocks using the specified memory allocation as temporary storage. Logical warp and lane identifiers are constructed from threadIdx.x. + */ + __device__ __forceinline__ WarpReduce( + TempStorage &temp_storage) ///< [in] Reference to memory allocation having layout type TempStorage + : + temp_storage(temp_storage.Alias()), + warp_id((LOGICAL_WARPS == 1) ? + 0 : + threadIdx.x / LOGICAL_WARP_THREADS), + lane_id(((LOGICAL_WARPS == 1) || (LOGICAL_WARP_THREADS == PtxArchProps::WARP_THREADS)) ? + LaneId() : + threadIdx.x % LOGICAL_WARP_THREADS) + {} + + + /** + * \brief Collective constructor using a private static allocation of shared memory as temporary storage. Threads are identified using the given warp and lane identifiers. + */ + __device__ __forceinline__ WarpReduce( + int warp_id, ///< [in] A suitable warp membership identifier + int lane_id) ///< [in] A lane identifier within the warp + : + temp_storage(PrivateStorage()), + warp_id(warp_id), + lane_id(lane_id) + {} + + + /** + * \brief Collective constructor using the specified memory allocation as temporary storage. Threads are identified using the given warp and lane identifiers. + */ + __device__ __forceinline__ WarpReduce( + TempStorage &temp_storage, ///< [in] Reference to memory allocation having layout type TempStorage + int warp_id, ///< [in] A suitable warp membership identifier + int lane_id) ///< [in] A lane identifier within the warp + : + temp_storage(temp_storage.Alias()), + warp_id(warp_id), + lane_id(lane_id) + {} + + + + //@} end member group + /******************************************************************//** + * \name Summation reductions + *********************************************************************/ + //@{ + + + /** + * \brief Computes a warp-wide sum in each active warp. The output is valid in warp lane0. + * + * \smemreuse + * + * The code snippet below illustrates four concurrent warp sum reductions within a block of + * 128 threads (one per each of the 32-thread warps). + * \par + * \code + * #include + * + * __global__ void ExampleKernel(...) + * { + * // Specialize WarpReduce for 4 warps on type int + * typedef cub::WarpReduce WarpReduce; + * + * // Allocate shared memory for WarpReduce + * __shared__ typename WarpReduce::TempStorage temp_storage; + * + * // Obtain one input item per thread + * int thread_data = ... + * + * // Return the warp-wide sums to each lane0 + * int aggregate = WarpReduce(temp_storage).Sum(thread_data); + * + * \endcode + * \par + * Suppose the set of input \p thread_data across the block of threads is 0, 1, 2, 3, ..., 127. + * The corresponding output \p aggregate in threads 0, 32, 64, and 96 will \p 496, \p 1520, + * \p 2544, and \p 3568, respectively (and is undefined in other threads). + * + */ + __device__ __forceinline__ T Sum( + T input) ///< [in] Calling thread's input + { + return InternalWarpReduce(temp_storage, warp_id, lane_id).Sum(input, LOGICAL_WARP_THREADS); + } + + /** + * \brief Computes a partially-full warp-wide sum in each active warp. The output is valid in warp lane0. + * + * All threads in each logical warp must agree on the same value for \p valid_items. Otherwise the result is undefined. + * + * \smemreuse + * + * The code snippet below illustrates a sum reduction within a single, partially-full + * block of 32 threads (one warp). + * \par + * \code + * #include + * + * __global__ void ExampleKernel(int *d_data, int valid_items) + * { + * // Specialize WarpReduce for a single warp on type int + * typedef cub::WarpReduce WarpReduce; + * + * // Allocate shared memory for WarpReduce + * __shared__ typename WarpReduce::TempStorage temp_storage; + * + * // Obtain one input item per thread if in range + * int thread_data; + * if (threadIdx.x < valid_items) + * thread_data = d_data[threadIdx.x]; + * + * // Return the warp-wide sums to each lane0 + * int aggregate = WarpReduce(temp_storage).Sum( + * thread_data, valid_items); + * + * \endcode + * \par + * Suppose the input \p d_data is 0, 1, 2, 3, 4, ... and \p valid_items + * is \p 4. The corresponding output \p aggregate in thread0 is \p 6 (and is + * undefined in other threads). + * + */ + __device__ __forceinline__ T Sum( + T input, ///< [in] Calling thread's input + int valid_items) ///< [in] Total number of valid items in the calling thread's logical warp (may be less than \p LOGICAL_WARP_THREADS) + { + // Determine if we don't need bounds checking + if (valid_items >= LOGICAL_WARP_THREADS) + { + return InternalWarpReduce(temp_storage, warp_id, lane_id).Sum(input, valid_items); + } + else + { + return InternalWarpReduce(temp_storage, warp_id, lane_id).Sum(input, valid_items); + } + } + + + /** + * \brief Computes a segmented sum in each active warp where segments are defined by head-flags. The sum of each segment is returned to the first lane in that segment (which always includes lane0). + * + * \smemreuse + * + * The code snippet below illustrates a head-segmented warp sum + * reduction within a block of 32 threads (one warp). + * \par + * \code + * #include + * + * __global__ void ExampleKernel(...) + * { + * // Specialize WarpReduce for a single warp on type int + * typedef cub::WarpReduce WarpReduce; + * + * // Allocate shared memory for WarpReduce + * __shared__ typename WarpReduce::TempStorage temp_storage; + * + * // Obtain one input item and flag per thread + * int thread_data = ... + * int head_flag = ... + * + * // Return the warp-wide sums to each lane0 + * int aggregate = WarpReduce(temp_storage).HeadSegmentedSum( + * thread_data, head_flag); + * + * \endcode + * \par + * Suppose the set of input \p thread_data and \p head_flag across the block of threads + * is 0, 1, 2, 3, ..., 31 and is 1, 0, 0, 0, 1, 0, 0, 0, ..., 1, 0, 0, 0, + * respectively. The corresponding output \p aggregate in threads 0, 4, 8, etc. will be + * \p 6, \p 22, \p 38, etc. (and is undefined in other threads). + * + * \tparam ReductionOp [inferred] Binary reduction operator type having member T operator()(const T &a, const T &b) + * + */ + template < + typename Flag> + __device__ __forceinline__ T HeadSegmentedSum( + T input, ///< [in] Calling thread's input + Flag head_flag) ///< [in] Head flag denoting whether or not \p input is the start of a new segment + { + return HeadSegmentedReduce(input, head_flag, cub::Sum()); + } + + + /** + * \brief Computes a segmented sum in each active warp where segments are defined by tail-flags. The sum of each segment is returned to the first lane in that segment (which always includes lane0). + * + * \smemreuse + * + * The code snippet below illustrates a tail-segmented warp sum + * reduction within a block of 32 threads (one warp). + * \par + * \code + * #include + * + * __global__ void ExampleKernel(...) + * { + * // Specialize WarpReduce for a single warp on type int + * typedef cub::WarpReduce WarpReduce; + * + * // Allocate shared memory for WarpReduce + * __shared__ typename WarpReduce::TempStorage temp_storage; + * + * // Obtain one input item and flag per thread + * int thread_data = ... + * int tail_flag = ... + * + * // Return the warp-wide sums to each lane0 + * int aggregate = WarpReduce(temp_storage).TailSegmentedSum( + * thread_data, tail_flag); + * + * \endcode + * \par + * Suppose the set of input \p thread_data and \p tail_flag across the block of threads + * is 0, 1, 2, 3, ..., 31 and is 0, 0, 0, 1, 0, 0, 0, 1, ..., 0, 0, 0, 1, + * respectively. The corresponding output \p aggregate in threads 0, 4, 8, etc. will be + * \p 6, \p 22, \p 38, etc. (and is undefined in other threads). + * + * \tparam ReductionOp [inferred] Binary reduction operator type having member T operator()(const T &a, const T &b) + */ + template < + typename Flag> + __device__ __forceinline__ T TailSegmentedSum( + T input, ///< [in] Calling thread's input + Flag tail_flag) ///< [in] Head flag denoting whether or not \p input is the start of a new segment + { + return TailSegmentedReduce(input, tail_flag, cub::Sum()); + } + + + + //@} end member group + /******************************************************************//** + * \name Generic reductions + *********************************************************************/ + //@{ + + /** + * \brief Computes a warp-wide reduction in each active warp using the specified binary reduction functor. The output is valid in warp lane0. + * + * Supports non-commutative reduction operators + * + * \smemreuse + * + * The code snippet below illustrates four concurrent warp max reductions within a block of + * 128 threads (one per each of the 32-thread warps). + * \par + * \code + * #include + * + * __global__ void ExampleKernel(...) + * { + * // Specialize WarpReduce for 4 warps on type int + * typedef cub::WarpReduce WarpReduce; + * + * // Allocate shared memory for WarpReduce + * __shared__ typename WarpReduce::TempStorage temp_storage; + * + * // Obtain one input item per thread + * int thread_data = ... + * + * // Return the warp-wide reductions to each lane0 + * int aggregate = WarpReduce(temp_storage).Reduce( + * thread_data, cub::Max()); + * + * \endcode + * \par + * Suppose the set of input \p thread_data across the block of threads is 0, 1, 2, 3, ..., 127. + * The corresponding output \p aggregate in threads 0, 32, 64, and 96 will \p 31, \p 63, + * \p 95, and \p 127, respectively (and is undefined in other threads). + * + * \tparam ReductionOp [inferred] Binary reduction operator type having member T operator()(const T &a, const T &b) + */ + template + __device__ __forceinline__ T Reduce( + T input, ///< [in] Calling thread's input + ReductionOp reduction_op) ///< [in] Binary reduction operator + { + return InternalWarpReduce(temp_storage, warp_id, lane_id).Reduce(input, LOGICAL_WARP_THREADS, reduction_op); + } + + /** + * \brief Computes a partially-full warp-wide reduction in each active warp using the specified binary reduction functor. The output is valid in warp lane0. + * + * All threads in each logical warp must agree on the same value for \p valid_items. Otherwise the result is undefined. + * + * Supports non-commutative reduction operators + * + * \smemreuse + * + * The code snippet below illustrates a max reduction within a single, partially-full + * block of 32 threads (one warp). + * \par + * \code + * #include + * + * __global__ void ExampleKernel(int *d_data, int valid_items) + * { + * // Specialize WarpReduce for a single warp on type int + * typedef cub::WarpReduce WarpReduce; + * + * // Allocate shared memory for WarpReduce + * __shared__ typename WarpReduce::TempStorage temp_storage; + * + * // Obtain one input item per thread if in range + * int thread_data; + * if (threadIdx.x < valid_items) + * thread_data = d_data[threadIdx.x]; + * + * // Return the warp-wide reductions to each lane0 + * int aggregate = WarpReduce(temp_storage).Reduce( + * thread_data, cub::Max(), valid_items); + * + * \endcode + * \par + * Suppose the input \p d_data is 0, 1, 2, 3, 4, ... and \p valid_items + * is \p 4. The corresponding output \p aggregate in thread0 is \p 3 (and is + * undefined in other threads). + * + * \tparam ReductionOp [inferred] Binary reduction operator type having member T operator()(const T &a, const T &b) + */ + template + __device__ __forceinline__ T Reduce( + T input, ///< [in] Calling thread's input + ReductionOp reduction_op, ///< [in] Binary reduction operator + int valid_items) ///< [in] Total number of valid items in the calling thread's logical warp (may be less than \p LOGICAL_WARP_THREADS) + { + // Determine if we don't need bounds checking + if (valid_items >= LOGICAL_WARP_THREADS) + { + return InternalWarpReduce(temp_storage, warp_id, lane_id).Reduce(input, valid_items, reduction_op); + } + else + { + return InternalWarpReduce(temp_storage, warp_id, lane_id).Reduce(input, valid_items, reduction_op); + } + } + + + /** + * \brief Computes a segmented reduction in each active warp where segments are defined by head-flags. The reduction of each segment is returned to the first lane in that segment (which always includes lane0). + * + * Supports non-commutative reduction operators + * + * \smemreuse + * + * The code snippet below illustrates a head-segmented warp max + * reduction within a block of 32 threads (one warp). + * \par + * \code + * #include + * + * __global__ void ExampleKernel(...) + * { + * // Specialize WarpReduce for a single warp on type int + * typedef cub::WarpReduce WarpReduce; + * + * // Allocate shared memory for WarpReduce + * __shared__ typename WarpReduce::TempStorage temp_storage; + * + * // Obtain one input item and flag per thread + * int thread_data = ... + * int head_flag = ... + * + * // Return the warp-wide reductions to each lane0 + * int aggregate = WarpReduce(temp_storage).HeadSegmentedReduce( + * thread_data, head_flag, cub::Max()); + * + * \endcode + * \par + * Suppose the set of input \p thread_data and \p head_flag across the block of threads + * is 0, 1, 2, 3, ..., 31 and is 1, 0, 0, 0, 1, 0, 0, 0, ..., 1, 0, 0, 0, + * respectively. The corresponding output \p aggregate in threads 0, 4, 8, etc. will be + * \p 3, \p 7, \p 11, etc. (and is undefined in other threads). + * + * \tparam ReductionOp [inferred] Binary reduction operator type having member T operator()(const T &a, const T &b) + */ + template < + typename ReductionOp, + typename Flag> + __device__ __forceinline__ T HeadSegmentedReduce( + T input, ///< [in] Calling thread's input + Flag head_flag, ///< [in] Head flag denoting whether or not \p input is the start of a new segment + ReductionOp reduction_op) ///< [in] Reduction operator + { + return InternalWarpReduce(temp_storage, warp_id, lane_id).template SegmentedReduce(input, head_flag, reduction_op); + } + + + /** + * \brief Computes a segmented reduction in each active warp where segments are defined by tail-flags. The reduction of each segment is returned to the first lane in that segment (which always includes lane0). + * + * Supports non-commutative reduction operators + * + * \smemreuse + * + * The code snippet below illustrates a tail-segmented warp max + * reduction within a block of 32 threads (one warp). + * \par + * \code + * #include + * + * __global__ void ExampleKernel(...) + * { + * // Specialize WarpReduce for a single warp on type int + * typedef cub::WarpReduce WarpReduce; + * + * // Allocate shared memory for WarpReduce + * __shared__ typename WarpReduce::TempStorage temp_storage; + * + * // Obtain one input item and flag per thread + * int thread_data = ... + * int tail_flag = ... + * + * // Return the warp-wide reductions to each lane0 + * int aggregate = WarpReduce(temp_storage).TailSegmentedReduce( + * thread_data, tail_flag, cub::Max()); + * + * \endcode + * \par + * Suppose the set of input \p thread_data and \p tail_flag across the block of threads + * is 0, 1, 2, 3, ..., 31 and is 0, 0, 0, 1, 0, 0, 0, 1, ..., 0, 0, 0, 1, + * respectively. The corresponding output \p aggregate in threads 0, 4, 8, etc. will be + * \p 3, \p 7, \p 11, etc. (and is undefined in other threads). + * + * \tparam ReductionOp [inferred] Binary reduction operator type having member T operator()(const T &a, const T &b) + */ + template < + typename ReductionOp, + typename Flag> + __device__ __forceinline__ T TailSegmentedReduce( + T input, ///< [in] Calling thread's input + Flag tail_flag, ///< [in] Tail flag denoting whether or not \p input is the end of the current segment + ReductionOp reduction_op) ///< [in] Reduction operator + { + return InternalWarpReduce(temp_storage, warp_id, lane_id).template SegmentedReduce(input, tail_flag, reduction_op); + } + + + + //@} end member group +}; + +/** @} */ // end group WarpModule + +} // CUB namespace +CUB_NS_POSTFIX // Optional outer namespace(s) diff --git a/lib/kokkos/TPL/cub/warp/warp_scan.cuh b/lib/kokkos/TPL/cub/warp/warp_scan.cuh new file mode 100755 index 0000000000..a588b52bd4 --- /dev/null +++ b/lib/kokkos/TPL/cub/warp/warp_scan.cuh @@ -0,0 +1,1297 @@ +/****************************************************************************** + * Copyright (c) 2011, Duane Merrill. All rights reserved. + * Copyright (c) 2011-2013, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the NVIDIA CORPORATION nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + ******************************************************************************/ + +/** + * \file + * The cub::WarpScan class provides [collective](index.html#sec0) methods for computing a parallel prefix scan of items partitioned across CUDA warp threads. + */ + +#pragma once + +#include "specializations/warp_scan_shfl.cuh" +#include "specializations/warp_scan_smem.cuh" +#include "../thread/thread_operators.cuh" +#include "../util_arch.cuh" +#include "../util_type.cuh" +#include "../util_namespace.cuh" + +/// Optional outer namespace(s) +CUB_NS_PREFIX + +/// CUB namespace +namespace cub { + +/** + * \addtogroup WarpModule + * @{ + */ + +/** + * \brief The WarpScan class provides [collective](index.html#sec0) methods for computing a parallel prefix scan of items partitioned across CUDA warp threads. ![](warp_scan_logo.png) + * + * \par Overview + * Given a list of input elements and a binary reduction operator, a [prefix scan](http://en.wikipedia.org/wiki/Prefix_sum) + * produces an output list where each element is computed to be the reduction + * of the elements occurring earlier in the input list. Prefix sum + * connotes a prefix scan with the addition operator. The term \em inclusive indicates + * that the ith output reduction incorporates the ith input. + * The term \em exclusive indicates the ith input is not incorporated into + * the ith output reduction. + * + * \tparam T The scan input/output element type + * \tparam LOGICAL_WARPS [optional] The number of "logical" warps performing concurrent warp scans. Default is 1. + * \tparam LOGICAL_WARP_THREADS [optional] The number of threads per "logical" warp (may be less than the number of hardware warp threads). Default is the warp size associated with the CUDA Compute Capability targeted by the compiler (e.g., 32 threads for SM20). + * + * \par Simple Examples + * \warpcollective{WarpScan} + * \par + * The code snippet below illustrates four concurrent warp prefix sums within a block of + * 128 threads (one per each of the 32-thread warps). + * \par + * \code + * #include + * + * __global__ void ExampleKernel(...) + * { + * // Specialize WarpScan for 4 warps on type int + * typedef cub::WarpScan WarpScan; + * + * // Allocate shared memory for WarpScan + * __shared__ typename WarpScan::TempStorage temp_storage; + * + * // Obtain one input item per thread + * int thread_data = ... + * + * // Compute warp-wide prefix sums + * WarpScan(temp_storage).ExclusiveSum(thread_data, thread_data); + * + * \endcode + * \par + * Suppose the set of input \p thread_data across the block of threads is 1, 1, 1, 1, .... + * The corresponding output \p thread_data in each of the four warps of threads will be + * 0, 1, 2, 3, ..., 31. + * + * \par + * The code snippet below illustrates a single warp prefix sum within a block of + * 128 threads. + * \par + * \code + * #include + * + * __global__ void ExampleKernel(...) + * { + * // Specialize WarpScan for one warp on type int + * typedef cub::WarpScan WarpScan; + * + * // Allocate shared memory for WarpScan + * __shared__ typename WarpScan::TempStorage temp_storage; + * ... + * + * // Only the first warp performs a prefix sum + * if (threadIdx.x < 32) + * { + * // Obtain one input item per thread + * int thread_data = ... + * + * // Compute warp-wide prefix sums + * WarpScan(temp_storage).ExclusiveSum(thread_data, thread_data); + * + * \endcode + * \par + * Suppose the set of input \p thread_data across the warp of threads is 1, 1, 1, 1, .... + * The corresponding output \p thread_data will be 0, 1, 2, 3, ..., 31. + * + * \par Usage and Performance Considerations + * - Supports "logical" warps smaller than the physical warp size (e.g., a logical warp of 8 threads) + * - The number of entrant threads must be an multiple of \p LOGICAL_WARP_THREADS + * - Warp scans are concurrent if more than one warp is participating + * - Uses special instructions when applicable (e.g., warp \p SHFL) + * - Uses synchronization-free communication between warp lanes when applicable + * - Zero bank conflicts for most types. + * - Computation is slightly more efficient (i.e., having lower instruction overhead) for: + * - Summation (vs. generic scan) + * - The architecture's warp size is a whole multiple of \p LOGICAL_WARP_THREADS + * + */ +template < + typename T, + int LOGICAL_WARPS = 1, + int LOGICAL_WARP_THREADS = PtxArchProps::WARP_THREADS> +class WarpScan +{ +private: + + /****************************************************************************** + * Constants and typedefs + ******************************************************************************/ + + enum + { + POW_OF_TWO = ((LOGICAL_WARP_THREADS & (LOGICAL_WARP_THREADS - 1)) == 0), + }; + + /// Internal specialization. Use SHFL-based reduction if (architecture is >= SM30) and ((only one logical warp) or (LOGICAL_WARP_THREADS is a power-of-two)) + typedef typename If<(CUB_PTX_ARCH >= 300) && ((LOGICAL_WARPS == 1) || POW_OF_TWO), + WarpScanShfl, + WarpScanSmem >::Type InternalWarpScan; + + /// Shared memory storage layout type for WarpScan + typedef typename InternalWarpScan::TempStorage _TempStorage; + + + /****************************************************************************** + * Thread fields + ******************************************************************************/ + + /// Shared storage reference + _TempStorage &temp_storage; + + /// Warp ID + int warp_id; + + /// Lane ID + int lane_id; + + + /****************************************************************************** + * Utility methods + ******************************************************************************/ + + /// Internal storage allocator + __device__ __forceinline__ _TempStorage& PrivateStorage() + { + __shared__ TempStorage private_storage; + return private_storage; + } + + +public: + + /// \smemstorage{WarpScan} + struct TempStorage : Uninitialized<_TempStorage> {}; + + + /******************************************************************//** + * \name Collective constructors + *********************************************************************/ + //@{ + + /** + * \brief Collective constructor for 1D thread blocks using a private static allocation of shared memory as temporary storage. Logical warp and lane identifiers are constructed from threadIdx.x. + */ + __device__ __forceinline__ WarpScan() + : + temp_storage(PrivateStorage()), + warp_id((LOGICAL_WARPS == 1) ? + 0 : + threadIdx.x / LOGICAL_WARP_THREADS), + lane_id(((LOGICAL_WARPS == 1) || (LOGICAL_WARP_THREADS == PtxArchProps::WARP_THREADS)) ? + LaneId() : + threadIdx.x % LOGICAL_WARP_THREADS) + {} + + + /** + * \brief Collective constructor for 1D thread blocks using the specified memory allocation as temporary storage. Logical warp and lane identifiers are constructed from threadIdx.x. + */ + __device__ __forceinline__ WarpScan( + TempStorage &temp_storage) ///< [in] Reference to memory allocation having layout type TempStorage + : + temp_storage(temp_storage.Alias()), + warp_id((LOGICAL_WARPS == 1) ? + 0 : + threadIdx.x / LOGICAL_WARP_THREADS), + lane_id(((LOGICAL_WARPS == 1) || (LOGICAL_WARP_THREADS == PtxArchProps::WARP_THREADS)) ? + LaneId() : + threadIdx.x % LOGICAL_WARP_THREADS) + {} + + + /** + * \brief Collective constructor using a private static allocation of shared memory as temporary storage. Threads are identified using the given warp and lane identifiers. + */ + __device__ __forceinline__ WarpScan( + int warp_id, ///< [in] A suitable warp membership identifier + int lane_id) ///< [in] A lane identifier within the warp + : + temp_storage(PrivateStorage()), + warp_id(warp_id), + lane_id(lane_id) + {} + + + /** + * \brief Collective constructor using the specified memory allocation as temporary storage. Threads are identified using the given warp and lane identifiers. + */ + __device__ __forceinline__ WarpScan( + TempStorage &temp_storage, ///< [in] Reference to memory allocation having layout type TempStorage + int warp_id, ///< [in] A suitable warp membership identifier + int lane_id) ///< [in] A lane identifier within the warp + : + temp_storage(temp_storage.Alias()), + warp_id(warp_id), + lane_id(lane_id) + {} + + + //@} end member group + /******************************************************************//** + * \name Inclusive prefix sums + *********************************************************************/ + //@{ + + + /** + * \brief Computes an inclusive prefix sum in each logical warp. + * + * \smemreuse + * + * The code snippet below illustrates four concurrent warp-wide inclusive prefix sums within a block of + * 128 threads (one per each of the 32-thread warps). + * \par + * \code + * #include + * + * __global__ void ExampleKernel(...) + * { + * // Specialize WarpScan for 4 warps on type int + * typedef cub::WarpScan WarpScan; + * + * // Allocate shared memory for WarpScan + * __shared__ typename WarpScan::TempStorage temp_storage; + * + * // Obtain one input item per thread + * int thread_data = ... + * + * // Compute inclusive warp-wide prefix sums + * WarpScan(temp_storage).InclusiveSum(thread_data, thread_data); + * + * \endcode + * \par + * Suppose the set of input \p thread_data across the block of threads is 1, 1, 1, 1, .... + * The corresponding output \p thread_data in each of the four warps of threads will be + * 1, 2, 3, ..., 32. + */ + __device__ __forceinline__ void InclusiveSum( + T input, ///< [in] Calling thread's input item. + T &output) ///< [out] Calling thread's output item. May be aliased with \p input. + { + InternalWarpScan(temp_storage, warp_id, lane_id).InclusiveSum(input, output); + } + + + /** + * \brief Computes an inclusive prefix sum in each logical warp. Also provides every thread with the warp-wide \p warp_aggregate of all inputs. + * + * The \p warp_aggregate is undefined in threads other than warp-lane0. + * + * \smemreuse + * + * The code snippet below illustrates four concurrent warp-wide inclusive prefix sums within a block of + * 128 threads (one per each of the 32-thread warps). + * \par + * \code + * #include + * + * __global__ void ExampleKernel(...) + * { + * // Specialize WarpScan for 4 warps on type int + * typedef cub::WarpScan WarpScan; + * + * // Allocate shared memory for WarpScan + * __shared__ typename WarpScan::TempStorage temp_storage; + * + * // Obtain one input item per thread + * int thread_data = ... + * + * // Compute inclusive warp-wide prefix sums + * int warp_aggregate; + * WarpScan(temp_storage).InclusiveSum(thread_data, thread_data, warp_aggregate); + * + * \endcode + * \par + * Suppose the set of input \p thread_data across the block of threads is 1, 1, 1, 1, .... + * The corresponding output \p thread_data in each of the four warps of threads will be + * 1, 2, 3, ..., 32. Furthermore, \p warp_aggregate for all threads in all warps will be \p 32. + */ + __device__ __forceinline__ void InclusiveSum( + T input, ///< [in] Calling thread's input item. + T &output, ///< [out] Calling thread's output item. May be aliased with \p input. + T &warp_aggregate) ///< [out] Warp-wide aggregate reduction of input items. + { + InternalWarpScan(temp_storage, warp_id, lane_id).InclusiveSum(input, output, warp_aggregate); + } + + + /** + * \brief Computes an inclusive prefix sum in each logical warp. Instead of using 0 as the warp-wide prefix, the call-back functor \p warp_prefix_op is invoked to provide the "seed" value that logically prefixes the warp's scan inputs. Also provides every thread with the warp-wide \p warp_aggregate of all inputs. + * + * The \p warp_aggregate is undefined in threads other than warp-lane0. + * + * The \p warp_prefix_op functor must implement a member function T operator()(T warp_aggregate). + * The functor's input parameter \p warp_aggregate is the same value also returned by the scan operation. + * The functor will be invoked by the entire warp of threads, however only the return value from + * lane0 is applied as the threadblock-wide prefix. Can be stateful. + * + * \smemreuse + * + * The code snippet below illustrates a single thread block of 32 threads (one warp) that progressively + * computes an inclusive prefix sum over multiple "tiles" of input using a + * prefix functor to maintain a running total between block-wide scans. Each tile consists + * of 32 integer items that are partitioned across the warp. + * \par + * \code + * #include + * + * // A stateful callback functor that maintains a running prefix to be applied + * // during consecutive scan operations. + * struct WarpPrefixOp + * { + * // Running prefix + * int running_total; + * + * // Constructor + * __device__ WarpPrefixOp(int running_total) : running_total(running_total) {} + * + * // Callback operator to be entered by the entire warp. Lane-0 is responsible + * // for returning a value for seeding the warp-wide scan. + * __device__ int operator()(int warp_aggregate) + * { + * int old_prefix = running_total; + * running_total += warp_aggregate; + * return old_prefix; + * } + * }; + * + * __global__ void ExampleKernel(int *d_data, int num_items, ...) + * { + * // Specialize WarpScan for one warp + * typedef cub::WarpScan WarpScan; + * + * // Allocate shared memory for WarpScan + * __shared__ typename WarpScan::TempStorage temp_storage; + * + * // Initialize running total + * WarpPrefixOp prefix_op(0); + * + * // Have the warp iterate over segments of items + * for (int block_offset = 0; block_offset < num_items; block_offset += 32) + * { + * // Load a segment of consecutive items + * int thread_data = d_data[block_offset]; + * + * // Collectively compute the warp-wide inclusive prefix sum + * int warp_aggregate; + * WarpScan(temp_storage).InclusiveSum( + * thread_data, thread_data, warp_aggregate, prefix_op); + * + * // Store scanned items to output segment + * d_data[block_offset] = thread_data; + * } + * \endcode + * \par + * Suppose the input \p d_data is 1, 1, 1, 1, 1, 1, 1, 1, .... + * The corresponding output for the first segment will be 1, 2, 3, ..., 32. + * The output for the second segment will be 33, 34, 35, ..., 64. Furthermore, + * the value \p 32 will be stored in \p warp_aggregate for all threads after each scan. + * + * \tparam WarpPrefixOp [inferred] Call-back functor type having member T operator()(T warp_aggregate) + */ + template + __device__ __forceinline__ void InclusiveSum( + T input, ///< [in] Calling thread's input item. + T &output, ///< [out] Calling thread's output item. May be aliased with \p input. + T &warp_aggregate, ///< [out] [warp-lane0 only] Warp-wide aggregate reduction of input items, exclusive of the \p warp_prefix_op value + WarpPrefixOp &warp_prefix_op) ///< [in-out] [warp-lane0 only] Call-back functor for specifying a warp-wide prefix to be applied to all inputs. + { + // Compute inclusive warp scan + InclusiveSum(input, output, warp_aggregate); + + // Compute warp-wide prefix from aggregate, then broadcast to other lanes + T prefix; + prefix = warp_prefix_op(warp_aggregate); + prefix = InternalWarpScan(temp_storage, warp_id, lane_id).Broadcast(prefix, 0); + + // Update output + output = prefix + output; + } + + //@} end member group + +private: + + /// Computes an exclusive prefix sum in each logical warp. + __device__ __forceinline__ void ExclusiveSum(T input, T &output, Int2Type is_primitive) + { + // Compute exclusive warp scan from inclusive warp scan + T inclusive; + InclusiveSum(input, inclusive); + output = inclusive - input; + } + + /// Computes an exclusive prefix sum in each logical warp. Specialized for non-primitive types. + __device__ __forceinline__ void ExclusiveSum(T input, T &output, Int2Type is_primitive) + { + // Delegate to regular scan for non-primitive types (because we won't be able to use subtraction) + T identity = T(); + ExclusiveScan(input, output, identity, Sum()); + } + + /// Computes an exclusive prefix sum in each logical warp. Also provides every thread with the warp-wide \p warp_aggregate of all inputs. + __device__ __forceinline__ void ExclusiveSum(T input, T &output, T &warp_aggregate, Int2Type is_primitive) + { + // Compute exclusive warp scan from inclusive warp scan + T inclusive; + InclusiveSum(input, inclusive, warp_aggregate); + output = inclusive - input; + } + + /// Computes an exclusive prefix sum in each logical warp. Also provides every thread with the warp-wide \p warp_aggregate of all inputs. Specialized for non-primitive types. + __device__ __forceinline__ void ExclusiveSum(T input, T &output, T &warp_aggregate, Int2Type is_primitive) + { + // Delegate to regular scan for non-primitive types (because we won't be able to use subtraction) + T identity = T(); + ExclusiveScan(input, output, identity, Sum(), warp_aggregate); + } + + /// Computes an exclusive prefix sum in each logical warp. Instead of using 0 as the warp-wide prefix, the call-back functor \p warp_prefix_op is invoked to provide the "seed" value that logically prefixes the warp's scan inputs. Also provides every thread with the warp-wide \p warp_aggregate of all inputs. + template + __device__ __forceinline__ void ExclusiveSum(T input, T &output, T &warp_aggregate, WarpPrefixOp &warp_prefix_op, Int2Type is_primitive) + { + // Compute exclusive warp scan from inclusive warp scan + T inclusive; + InclusiveSum(input, inclusive, warp_aggregate, warp_prefix_op); + output = inclusive - input; + } + + /// Computes an exclusive prefix sum in each logical warp. Instead of using 0 as the warp-wide prefix, the call-back functor \p warp_prefix_op is invoked to provide the "seed" value that logically prefixes the warp's scan inputs. Also provides every thread with the warp-wide \p warp_aggregate of all inputs. Specialized for non-primitive types. + template + __device__ __forceinline__ void ExclusiveSum(T input, T &output, T &warp_aggregate, WarpPrefixOp &warp_prefix_op, Int2Type is_primitive) + { + // Delegate to regular scan for non-primitive types (because we won't be able to use subtraction) + T identity = T(); + ExclusiveScan(input, output, identity, Sum(), warp_aggregate, warp_prefix_op); + } + +public: + + + /******************************************************************//** + * \name Exclusive prefix sums + *********************************************************************/ + //@{ + + + /** + * \brief Computes an exclusive prefix sum in each logical warp. + * + * This operation assumes the value of obtained by the T's default + * constructor (or by zero-initialization if no user-defined default + * constructor exists) is suitable as the identity value "zero" for + * addition. + * + * \smemreuse + * + * The code snippet below illustrates four concurrent warp-wide exclusive prefix sums within a block of + * 128 threads (one per each of the 32-thread warps). + * \par + * \code + * #include + * + * __global__ void ExampleKernel(...) + * { + * // Specialize WarpScan for 4 warps on type int + * typedef cub::WarpScan WarpScan; + * + * // Allocate shared memory for WarpScan + * __shared__ typename WarpScan::TempStorage temp_storage; + * + * // Obtain one input item per thread + * int thread_data = ... + * + * // Compute exclusive warp-wide prefix sums + * WarpScan(temp_storage).ExclusiveSum(thread_data, thread_data); + * + * \endcode + * \par + * Suppose the set of input \p thread_data across the block of threads is 1, 1, 1, 1, .... + * The corresponding output \p thread_data in each of the four warps of threads will be + * 0, 1, 2, ..., 31. + * + */ + __device__ __forceinline__ void ExclusiveSum( + T input, ///< [in] Calling thread's input item. + T &output) ///< [out] Calling thread's output item. May be aliased with \p input. + { + ExclusiveSum(input, output, Int2Type::PRIMITIVE>()); + } + + + /** + * \brief Computes an exclusive prefix sum in each logical warp. Also provides every thread with the warp-wide \p warp_aggregate of all inputs. + * + * This operation assumes the value of obtained by the T's default + * constructor (or by zero-initialization if no user-defined default + * constructor exists) is suitable as the identity value "zero" for + * addition. + * + * \smemreuse + * + * The code snippet below illustrates four concurrent warp-wide exclusive prefix sums within a block of + * 128 threads (one per each of the 32-thread warps). + * \par + * \code + * #include + * + * __global__ void ExampleKernel(...) + * { + * // Specialize WarpScan for 4 warps on type int + * typedef cub::WarpScan WarpScan; + * + * // Allocate shared memory for WarpScan + * __shared__ typename WarpScan::TempStorage temp_storage; + * + * // Obtain one input item per thread + * int thread_data = ... + * + * // Compute exclusive warp-wide prefix sums + * int warp_aggregate; + * WarpScan(temp_storage).ExclusiveSum(thread_data, thread_data, warp_aggregate); + * + * \endcode + * \par + * Suppose the set of input \p thread_data across the block of threads is 1, 1, 1, 1, .... + * The corresponding output \p thread_data in each of the four warps of threads will be + * 0, 1, 2, ..., 31. Furthermore, \p warp_aggregate for all threads in all warps will be \p 32. + */ + __device__ __forceinline__ void ExclusiveSum( + T input, ///< [in] Calling thread's input item. + T &output, ///< [out] Calling thread's output item. May be aliased with \p input. + T &warp_aggregate) ///< [out] Warp-wide aggregate reduction of input items. + { + ExclusiveSum(input, output, warp_aggregate, Int2Type::PRIMITIVE>()); + } + + + /** + * \brief Computes an exclusive prefix sum in each logical warp. Instead of using 0 as the warp-wide prefix, the call-back functor \p warp_prefix_op is invoked to provide the "seed" value that logically prefixes the warp's scan inputs. Also provides every thread with the warp-wide \p warp_aggregate of all inputs. + * + * This operation assumes the value of obtained by the T's default + * constructor (or by zero-initialization if no user-defined default + * constructor exists) is suitable as the identity value "zero" for + * addition. + * + * The \p warp_prefix_op functor must implement a member function T operator()(T warp_aggregate). + * The functor's input parameter \p warp_aggregate is the same value also returned by the scan operation. + * The functor will be invoked by the entire warp of threads, however only the return value from + * lane0 is applied as the threadblock-wide prefix. Can be stateful. + * + * \smemreuse + * + * The code snippet below illustrates a single thread block of 32 threads (one warp) that progressively + * computes an exclusive prefix sum over multiple "tiles" of input using a + * prefix functor to maintain a running total between block-wide scans. Each tile consists + * of 32 integer items that are partitioned across the warp. + * \par + * \code + * #include + * + * // A stateful callback functor that maintains a running prefix to be applied + * // during consecutive scan operations. + * struct WarpPrefixOp + * { + * // Running prefix + * int running_total; + * + * // Constructor + * __device__ WarpPrefixOp(int running_total) : running_total(running_total) {} + * + * // Callback operator to be entered by the entire warp. Lane-0 is responsible + * // for returning a value for seeding the warp-wide scan. + * __device__ int operator()(int warp_aggregate) + * { + * int old_prefix = running_total; + * running_total += warp_aggregate; + * return old_prefix; + * } + * }; + * + * __global__ void ExampleKernel(int *d_data, int num_items, ...) + * { + * // Specialize WarpScan for one warp + * typedef cub::WarpScan WarpScan; + * + * // Allocate shared memory for WarpScan + * __shared__ typename WarpScan::TempStorage temp_storage; + * + * // Initialize running total + * WarpPrefixOp prefix_op(0); + * + * // Have the warp iterate over segments of items + * for (int block_offset = 0; block_offset < num_items; block_offset += 32) + * { + * // Load a segment of consecutive items + * int thread_data = d_data[block_offset]; + * + * // Collectively compute the warp-wide exclusive prefix sum + * int warp_aggregate; + * WarpScan(temp_storage).ExclusiveSum( + * thread_data, thread_data, warp_aggregate, prefix_op); + * + * // Store scanned items to output segment + * d_data[block_offset] = thread_data; + * } + * \endcode + * \par + * Suppose the input \p d_data is 1, 1, 1, 1, 1, 1, 1, 1, .... + * The corresponding output for the first segment will be 0, 1, 2, ..., 31. + * The output for the second segment will be 32, 33, 34, ..., 63. Furthermore, + * the value \p 32 will be stored in \p warp_aggregate for all threads after each scan. + * + * \tparam WarpPrefixOp [inferred] Call-back functor type having member T operator()(T warp_aggregate) + */ + template + __device__ __forceinline__ void ExclusiveSum( + T input, ///< [in] Calling thread's input item. + T &output, ///< [out] Calling thread's output item. May be aliased with \p input. + T &warp_aggregate, ///< [out] [warp-lane0 only] Warp-wide aggregate reduction of input items (exclusive of the \p warp_prefix_op value). + WarpPrefixOp &warp_prefix_op) ///< [in-out] [warp-lane0 only] Call-back functor for specifying a warp-wide prefix to be applied to all inputs. + { + ExclusiveSum(input, output, warp_aggregate, warp_prefix_op, Int2Type::PRIMITIVE>()); + } + + + //@} end member group + /******************************************************************//** + * \name Inclusive prefix scans + *********************************************************************/ + //@{ + + /** + * \brief Computes an inclusive prefix sum using the specified binary scan functor in each logical warp. + * + * Supports non-commutative scan operators. + * + * \smemreuse + * + * The code snippet below illustrates four concurrent warp-wide inclusive prefix max scans within a block of + * 128 threads (one per each of the 32-thread warps). + * \par + * \code + * #include + * + * __global__ void ExampleKernel(...) + * { + * // Specialize WarpScan for 4 warps on type int + * typedef cub::WarpScan WarpScan; + * + * // Allocate shared memory for WarpScan + * __shared__ typename WarpScan::TempStorage temp_storage; + * + * // Obtain one input item per thread + * int thread_data = ... + * + * // Compute inclusive warp-wide prefix max scans + * WarpScan(temp_storage).InclusiveScan(thread_data, thread_data, cub::Max()); + * + * \endcode + * \par + * Suppose the set of input \p thread_data across the block of threads is 0, -1, 2, -3, ..., 126, -127. + * The corresponding output \p thread_data in the first warp would be + * 0, 0, 2, 2, ..., 30, 30, the output for the second warp would be 32, 32, 34, 34, ..., 62, 62, etc. + * + * \tparam ScanOp [inferred] Binary scan operator type having member T operator()(const T &a, const T &b) + */ + template + __device__ __forceinline__ void InclusiveScan( + T input, ///< [in] Calling thread's input item. + T &output, ///< [out] Calling thread's output item. May be aliased with \p input. + ScanOp scan_op) ///< [in] Binary scan operator + { + InternalWarpScan(temp_storage, warp_id, lane_id).InclusiveScan(input, output, scan_op); + } + + + /** + * \brief Computes an inclusive prefix sum using the specified binary scan functor in each logical warp. Also provides every thread with the warp-wide \p warp_aggregate of all inputs. + * + * Supports non-commutative scan operators. + * + * \smemreuse + * + * The code snippet below illustrates four concurrent warp-wide inclusive prefix max scans within a block of + * 128 threads (one per each of the 32-thread warps). + * \par + * \code + * #include + * + * __global__ void ExampleKernel(...) + * { + * // Specialize WarpScan for 4 warps on type int + * typedef cub::WarpScan WarpScan; + * + * // Allocate shared memory for WarpScan + * __shared__ typename WarpScan::TempStorage temp_storage; + * + * // Obtain one input item per thread + * int thread_data = ... + * + * // Compute inclusive warp-wide prefix max scans + * int warp_aggregate; + * WarpScan(temp_storage).InclusiveScan( + * thread_data, thread_data, cub::Max(), warp_aggregate); + * + * \endcode + * \par + * Suppose the set of input \p thread_data across the block of threads is 0, -1, 2, -3, ..., 126, -127. + * The corresponding output \p thread_data in the first warp would be + * 0, 0, 2, 2, ..., 30, 30, the output for the second warp would be 32, 32, 34, 34, ..., 62, 62, etc. + * Furthermore, \p warp_aggregate would be assigned \p 30 for threads in the first warp, \p 62 for threads + * in the second warp, etc. + * + * \tparam ScanOp [inferred] Binary scan operator type having member T operator()(const T &a, const T &b) + */ + template + __device__ __forceinline__ void InclusiveScan( + T input, ///< [in] Calling thread's input item. + T &output, ///< [out] Calling thread's output item. May be aliased with \p input. + ScanOp scan_op, ///< [in] Binary scan operator + T &warp_aggregate) ///< [out] Warp-wide aggregate reduction of input items. + { + InternalWarpScan(temp_storage, warp_id, lane_id).InclusiveScan(input, output, scan_op, warp_aggregate); + } + + + /** + * \brief Computes an inclusive prefix sum using the specified binary scan functor in each logical warp. The call-back functor \p warp_prefix_op is invoked to provide the "seed" value that logically prefixes the warp's scan inputs. Also provides every thread with the warp-wide \p warp_aggregate of all inputs. + * + * The \p warp_prefix_op functor must implement a member function T operator()(T warp_aggregate). + * The functor's input parameter \p warp_aggregate is the same value also returned by the scan operation. + * The functor will be invoked by the entire warp of threads, however only the return value from + * lane0 is applied as the threadblock-wide prefix. Can be stateful. + * + * Supports non-commutative scan operators. + * + * \smemreuse + * + * The code snippet below illustrates a single thread block of 32 threads (one warp) that progressively + * computes an inclusive prefix max scan over multiple "tiles" of input using a + * prefix functor to maintain a running total between block-wide scans. Each tile consists + * of 32 integer items that are partitioned across the warp. + * \par + * \code + * #include + * + * // A stateful callback functor that maintains a running prefix to be applied + * // during consecutive scan operations. + * struct WarpPrefixOp + * { + * // Running prefix + * int running_total; + * + * // Constructor + * __device__ WarpPrefixOp(int running_total) : running_total(running_total) {} + * + * // Callback operator to be entered by the entire warp. Lane-0 is responsible + * // for returning a value for seeding the warp-wide scan. + * __device__ int operator()(int warp_aggregate) + * { + * int old_prefix = running_total; + * running_total = (warp_aggregate > old_prefix) ? warp_aggregate : old_prefix; + * return old_prefix; + * } + * }; + * + * __global__ void ExampleKernel(int *d_data, int num_items, ...) + * { + * // Specialize WarpScan for one warp + * typedef cub::WarpScan WarpScan; + * + * // Allocate shared memory for WarpScan + * __shared__ typename WarpScan::TempStorage temp_storage; + * + * // Initialize running total + * WarpPrefixOp prefix_op(0); + * + * // Have the warp iterate over segments of items + * for (int block_offset = 0; block_offset < num_items; block_offset += 32) + * { + * // Load a segment of consecutive items + * int thread_data = d_data[block_offset]; + * + * // Collectively compute the warp-wide inclusive prefix max scan + * int warp_aggregate; + * WarpScan(temp_storage).InclusiveScan( + * thread_data, thread_data, cub::Max(), warp_aggregate, prefix_op); + * + * // Store scanned items to output segment + * d_data[block_offset] = thread_data; + * } + * \endcode + * \par + * Suppose the input \p d_data is 0, -1, 2, -3, 4, -5, .... + * The corresponding output for the first segment will be 0, 0, 2, 2, ..., 30, 30. + * The output for the second segment will be 32, 32, 34, 34, ..., 62, 62. Furthermore, + * \p block_aggregate will be assigned \p 30 in all threads after the first scan, assigned \p 62 after the second + * scan, etc. + * + * \tparam ScanOp [inferred] Binary scan operator type having member T operator()(const T &a, const T &b) + * \tparam WarpPrefixOp [inferred] Call-back functor type having member T operator()(T warp_aggregate) + */ + template < + typename ScanOp, + typename WarpPrefixOp> + __device__ __forceinline__ void InclusiveScan( + T input, ///< [in] Calling thread's input item. + T &output, ///< [out] Calling thread's output item. May be aliased with \p input. + ScanOp scan_op, ///< [in] Binary scan operator + T &warp_aggregate, ///< [out] [warp-lane0 only] Warp-wide aggregate reduction of input items (exclusive of the \p warp_prefix_op value). + WarpPrefixOp &warp_prefix_op) ///< [in-out] [warp-lane0 only] Call-back functor for specifying a warp-wide prefix to be applied to all inputs. + { + // Compute inclusive warp scan + InclusiveScan(input, output, scan_op, warp_aggregate); + + // Compute warp-wide prefix from aggregate, then broadcast to other lanes + T prefix; + prefix = warp_prefix_op(warp_aggregate); + prefix = InternalWarpScan(temp_storage, warp_id, lane_id).Broadcast(prefix, 0); + + // Update output + output = scan_op(prefix, output); + } + + + //@} end member group + /******************************************************************//** + * \name Exclusive prefix scans + *********************************************************************/ + //@{ + + /** + * \brief Computes an exclusive prefix scan using the specified binary scan functor in each logical warp. + * + * Supports non-commutative scan operators. + * + * \smemreuse + * + * The code snippet below illustrates four concurrent warp-wide exclusive prefix max scans within a block of + * 128 threads (one per each of the 32-thread warps). + * \par + * \code + * #include + * + * __global__ void ExampleKernel(...) + * { + * // Specialize WarpScan for 4 warps on type int + * typedef cub::WarpScan WarpScan; + * + * // Allocate shared memory for WarpScan + * __shared__ typename WarpScan::TempStorage temp_storage; + * + * // Obtain one input item per thread + * int thread_data = ... + * + * // Compute exclusive warp-wide prefix max scans + * WarpScan(temp_storage).ExclusiveScan(thread_data, thread_data, INT_MIN, cub::Max()); + * + * \endcode + * \par + * Suppose the set of input \p thread_data across the block of threads is 0, -1, 2, -3, ..., 126, -127. + * The corresponding output \p thread_data in the first warp would be + * INT_MIN, 0, 0, 2, ..., 28, 30, the output for the second warp would be 30, 32, 32, 34, ..., 60, 62, etc. + * + * \tparam ScanOp [inferred] Binary scan operator type having member T operator()(const T &a, const T &b) + */ + template + __device__ __forceinline__ void ExclusiveScan( + T input, ///< [in] Calling thread's input item. + T &output, ///< [out] Calling thread's output item. May be aliased with \p input. + T identity, ///< [in] Identity value + ScanOp scan_op) ///< [in] Binary scan operator + { + InternalWarpScan(temp_storage, warp_id, lane_id).ExclusiveScan(input, output, identity, scan_op); + } + + + /** + * \brief Computes an exclusive prefix scan using the specified binary scan functor in each logical warp. Also provides every thread with the warp-wide \p warp_aggregate of all inputs. + * + * Supports non-commutative scan operators. + * + * \smemreuse + * + * The code snippet below illustrates four concurrent warp-wide exclusive prefix max scans within a block of + * 128 threads (one per each of the 32-thread warps). + * \par + * \code + * #include + * + * __global__ void ExampleKernel(...) + * { + * // Specialize WarpScan for 4 warps on type int + * typedef cub::WarpScan WarpScan; + * + * // Allocate shared memory for WarpScan + * __shared__ typename WarpScan::TempStorage temp_storage; + * + * // Obtain one input item per thread + * int thread_data = ... + * + * // Compute exclusive warp-wide prefix max scans + * WarpScan(temp_storage).ExclusiveScan(thread_data, thread_data, INT_MIN, cub::Max()); + * + * \endcode + * \par + * Suppose the set of input \p thread_data across the block of threads is 0, -1, 2, -3, ..., 126, -127. + * The corresponding output \p thread_data in the first warp would be + * INT_MIN, 0, 0, 2, ..., 28, 30, the output for the second warp would be 30, 32, 32, 34, ..., 60, 62, etc. + * Furthermore, \p warp_aggregate would be assigned \p 30 for threads in the first warp, \p 62 for threads + * in the second warp, etc. + * + * \tparam ScanOp [inferred] Binary scan operator type having member T operator()(const T &a, const T &b) + */ + template + __device__ __forceinline__ void ExclusiveScan( + T input, ///< [in] Calling thread's input item. + T &output, ///< [out] Calling thread's output item. May be aliased with \p input. + T identity, ///< [in] Identity value + ScanOp scan_op, ///< [in] Binary scan operator + T &warp_aggregate) ///< [out] Warp-wide aggregate reduction of input items. + { + InternalWarpScan(temp_storage, warp_id, lane_id).ExclusiveScan(input, output, identity, scan_op, warp_aggregate); + } + + + /** + * \brief Computes an exclusive prefix scan using the specified binary scan functor in each logical warp. The call-back functor \p warp_prefix_op is invoked to provide the "seed" value that logically prefixes the warp's scan inputs. Also provides every thread with the warp-wide \p warp_aggregate of all inputs. + * + * The \p warp_prefix_op functor must implement a member function T operator()(T warp_aggregate). + * The functor's input parameter \p warp_aggregate is the same value also returned by the scan operation. + * The functor will be invoked by the entire warp of threads, however only the return value from + * lane0 is applied as the threadblock-wide prefix. Can be stateful. + * + * Supports non-commutative scan operators. + * + * \smemreuse + * + * The code snippet below illustrates a single thread block of 32 threads (one warp) that progressively + * computes an exclusive prefix max scan over multiple "tiles" of input using a + * prefix functor to maintain a running total between block-wide scans. Each tile consists + * of 32 integer items that are partitioned across the warp. + * \par + * \code + * #include + * + * // A stateful callback functor that maintains a running prefix to be applied + * // during consecutive scan operations. + * struct WarpPrefixOp + * { + * // Running prefix + * int running_total; + * + * // Constructor + * __device__ WarpPrefixOp(int running_total) : running_total(running_total) {} + * + * // Callback operator to be entered by the entire warp. Lane-0 is responsible + * // for returning a value for seeding the warp-wide scan. + * __device__ int operator()(int warp_aggregate) + * { + * int old_prefix = running_total; + * running_total = (warp_aggregate > old_prefix) ? warp_aggregate : old_prefix; + * return old_prefix; + * } + * }; + * + * __global__ void ExampleKernel(int *d_data, int num_items, ...) + * { + * // Specialize WarpScan for one warp + * typedef cub::WarpScan WarpScan; + * + * // Allocate shared memory for WarpScan + * __shared__ typename WarpScan::TempStorage temp_storage; + * + * // Initialize running total + * WarpPrefixOp prefix_op(INT_MIN); + * + * // Have the warp iterate over segments of items + * for (int block_offset = 0; block_offset < num_items; block_offset += 32) + * { + * // Load a segment of consecutive items + * int thread_data = d_data[block_offset]; + * + * // Collectively compute the warp-wide exclusive prefix max scan + * int warp_aggregate; + * WarpScan(temp_storage).ExclusiveScan( + * thread_data, thread_data, INT_MIN, cub::Max(), warp_aggregate, prefix_op); + * + * // Store scanned items to output segment + * d_data[block_offset] = thread_data; + * } + * \endcode + * \par + * Suppose the input \p d_data is 0, -1, 2, -3, 4, -5, .... + * The corresponding output for the first segment will be INT_MIN, 0, 0, 2, ..., 28, 30. + * The output for the second segment will be 30, 32, 32, 34, ..., 60, 62. Furthermore, + * \p block_aggregate will be assigned \p 30 in all threads after the first scan, assigned \p 62 after the second + * scan, etc. + * + * \tparam ScanOp [inferred] Binary scan operator type having member T operator()(const T &a, const T &b) + * \tparam WarpPrefixOp [inferred] Call-back functor type having member T operator()(T warp_aggregate) + */ + template < + typename ScanOp, + typename WarpPrefixOp> + __device__ __forceinline__ void ExclusiveScan( + T input, ///< [in] Calling thread's input item. + T &output, ///< [out] Calling thread's output item. May be aliased with \p input. + T identity, ///< [in] Identity value + ScanOp scan_op, ///< [in] Binary scan operator + T &warp_aggregate, ///< [out] [warp-lane0 only] Warp-wide aggregate reduction of input items (exclusive of the \p warp_prefix_op value). + WarpPrefixOp &warp_prefix_op) ///< [in-out] [warp-lane0 only] Call-back functor for specifying a warp-wide prefix to be applied to all inputs. + { + // Exclusive warp scan + ExclusiveScan(input, output, identity, scan_op, warp_aggregate); + + // Compute warp-wide prefix from aggregate, then broadcast to other lanes + T prefix = warp_prefix_op(warp_aggregate); + prefix = InternalWarpScan(temp_storage, warp_id, lane_id).Broadcast(prefix, 0); + + // Update output + output = (lane_id == 0) ? + prefix : + scan_op(prefix, output); + } + + + //@} end member group + /******************************************************************//** + * \name Identityless exclusive prefix scans + *********************************************************************/ + //@{ + + + /** + * \brief Computes an exclusive prefix scan using the specified binary scan functor in each logical warp. Because no identity value is supplied, the \p output computed for warp-lane0 is undefined. + * + * Supports non-commutative scan operators. + * + * \smemreuse + * + * The code snippet below illustrates four concurrent warp-wide exclusive prefix max scans within a block of + * 128 threads (one per each of the 32-thread warps). + * \par + * \code + * #include + * + * __global__ void ExampleKernel(...) + * { + * // Specialize WarpScan for 4 warps on type int + * typedef cub::WarpScan WarpScan; + * + * // Allocate shared memory for WarpScan + * __shared__ typename WarpScan::TempStorage temp_storage; + * + * // Obtain one input item per thread + * int thread_data = ... + * + * // Compute exclusive warp-wide prefix max scans + * WarpScan(temp_storage).ExclusiveScan(thread_data, thread_data, cub::Max()); + * + * \endcode + * \par + * Suppose the set of input \p thread_data across the block of threads is 0, -1, 2, -3, ..., 126, -127. + * The corresponding output \p thread_data in the first warp would be + * ?, 0, 0, 2, ..., 28, 30, the output for the second warp would be ?, 32, 32, 34, ..., 60, 62, etc. + * (The output \p thread_data in each warp lane0 is undefined.) + * + * \tparam ScanOp [inferred] Binary scan operator type having member T operator()(const T &a, const T &b) + */ + template + __device__ __forceinline__ void ExclusiveScan( + T input, ///< [in] Calling thread's input item. + T &output, ///< [out] Calling thread's output item. May be aliased with \p input. + ScanOp scan_op) ///< [in] Binary scan operator + { + InternalWarpScan(temp_storage, warp_id, lane_id).ExclusiveScan(input, output, scan_op); + } + + + /** + * \brief Computes an exclusive prefix scan using the specified binary scan functor in each logical warp. Because no identity value is supplied, the \p output computed for warp-lane0 is undefined. Also provides every thread with the warp-wide \p warp_aggregate of all inputs. + * + * Supports non-commutative scan operators. + * + * \smemreuse + * + * The code snippet below illustrates four concurrent warp-wide exclusive prefix max scans within a block of + * 128 threads (one per each of the 32-thread warps). + * \par + * \code + * #include + * + * __global__ void ExampleKernel(...) + * { + * // Specialize WarpScan for 4 warps on type int + * typedef cub::WarpScan WarpScan; + * + * // Allocate shared memory for WarpScan + * __shared__ typename WarpScan::TempStorage temp_storage; + * + * // Obtain one input item per thread + * int thread_data = ... + * + * // Compute exclusive warp-wide prefix max scans + * WarpScan(temp_storage).ExclusiveScan(thread_data, thread_data, cub::Max()); + * + * \endcode + * \par + * Suppose the set of input \p thread_data across the block of threads is 0, -1, 2, -3, ..., 126, -127. + * The corresponding output \p thread_data in the first warp would be + * ?, 0, 0, 2, ..., 28, 30, the output for the second warp would be ?, 32, 32, 34, ..., 60, 62, etc. + * (The output \p thread_data in each warp lane0 is undefined.) Furthermore, \p warp_aggregate would be assigned \p 30 for threads in the first warp, \p 62 for threads + * in the second warp, etc. + * + * \tparam ScanOp [inferred] Binary scan operator type having member T operator()(const T &a, const T &b) + */ + template + __device__ __forceinline__ void ExclusiveScan( + T input, ///< [in] Calling thread's input item. + T &output, ///< [out] Calling thread's output item. May be aliased with \p input. + ScanOp scan_op, ///< [in] Binary scan operator + T &warp_aggregate) ///< [out] Warp-wide aggregate reduction of input items. + { + InternalWarpScan(temp_storage, warp_id, lane_id).ExclusiveScan(input, output, scan_op, warp_aggregate); + } + + + /** + * \brief Computes an exclusive prefix scan using the specified binary scan functor in each logical warp. The \p warp_prefix_op value from thread-thread-lane0 is applied to all scan outputs. Also computes the warp-wide \p warp_aggregate of all inputs for thread-thread-lane0. + * + * The \p warp_prefix_op functor must implement a member function T operator()(T warp_aggregate). + * The functor's input parameter \p warp_aggregate is the same value also returned by the scan operation. + * The functor will be invoked by the entire warp of threads, however only the return value from + * lane0 is applied as the threadblock-wide prefix. Can be stateful. + * + * Supports non-commutative scan operators. + * + * \smemreuse + * + * The code snippet below illustrates a single thread block of 32 threads (one warp) that progressively + * computes an exclusive prefix max scan over multiple "tiles" of input using a + * prefix functor to maintain a running total between block-wide scans. Each tile consists + * of 32 integer items that are partitioned across the warp. + * \par + * \code + * #include + * + * // A stateful callback functor that maintains a running prefix to be applied + * // during consecutive scan operations. + * struct WarpPrefixOp + * { + * // Running prefix + * int running_total; + * + * // Constructor + * __device__ WarpPrefixOp(int running_total) : running_total(running_total) {} + * + * // Callback operator to be entered by the entire warp. Lane-0 is responsible + * // for returning a value for seeding the warp-wide scan. + * __device__ int operator()(int warp_aggregate) + * { + * int old_prefix = running_total; + * running_total = (warp_aggregate > old_prefix) ? warp_aggregate : old_prefix; + * return old_prefix; + * } + * }; + * + * __global__ void ExampleKernel(int *d_data, int num_items, ...) + * { + * // Specialize WarpScan for one warp + * typedef cub::WarpScan WarpScan; + * + * // Allocate shared memory for WarpScan + * __shared__ typename WarpScan::TempStorage temp_storage; + * + * // Initialize running total + * WarpPrefixOp prefix_op(INT_MIN); + * + * // Have the warp iterate over segments of items + * for (int block_offset = 0; block_offset < num_items; block_offset += 32) + * { + * // Load a segment of consecutive items + * int thread_data = d_data[block_offset]; + * + * // Collectively compute the warp-wide exclusive prefix max scan + * int warp_aggregate; + * WarpScan(temp_storage).ExclusiveScan( + * thread_data, thread_data, INT_MIN, cub::Max(), warp_aggregate, prefix_op); + * + * // Store scanned items to output segment + * d_data[block_offset] = thread_data; + * } + * \endcode + * \par + * Suppose the input \p d_data is 0, -1, 2, -3, 4, -5, .... + * The corresponding output for the first segment will be INT_MIN, 0, 0, 2, ..., 28, 30. + * The output for the second segment will be 30, 32, 32, 34, ..., 60, 62. Furthermore, + * \p block_aggregate will be assigned \p 30 in all threads after the first scan, assigned \p 62 after the second + * scan, etc. + * + * \tparam ScanOp [inferred] Binary scan operator type having member T operator()(const T &a, const T &b) + * \tparam WarpPrefixOp [inferred] Call-back functor type having member T operator()(T warp_aggregate) + */ + template < + typename ScanOp, + typename WarpPrefixOp> + __device__ __forceinline__ void ExclusiveScan( + T input, ///< [in] Calling thread's input item. + T &output, ///< [out] Calling thread's output item. May be aliased with \p input. + ScanOp scan_op, ///< [in] Binary scan operator + T &warp_aggregate, ///< [out] [warp-lane0 only] Warp-wide aggregate reduction of input items (exclusive of the \p warp_prefix_op value). + WarpPrefixOp &warp_prefix_op) ///< [in-out] [warp-lane0 only] Call-back functor for specifying a warp-wide prefix to be applied to all inputs. + { + // Exclusive warp scan + ExclusiveScan(input, output, scan_op, warp_aggregate); + + // Compute warp-wide prefix from aggregate, then broadcast to other lanes + T prefix = warp_prefix_op(warp_aggregate); + prefix = InternalWarpScan(temp_storage, warp_id, lane_id).Broadcast(prefix, 0); + + // Update output with prefix + output = (lane_id == 0) ? + prefix : + scan_op(prefix, output); + } + + //@} end member group +}; + +/** @} */ // end group WarpModule + +} // CUB namespace +CUB_NS_POSTFIX // Optional outer namespace(s) diff --git a/lib/kokkos/algorithms/src/Kokkos_Random.hpp b/lib/kokkos/algorithms/src/Kokkos_Random.hpp new file mode 100755 index 0000000000..903bc4eb0e --- /dev/null +++ b/lib/kokkos/algorithms/src/Kokkos_Random.hpp @@ -0,0 +1,1691 @@ +/* +//@HEADER +// ************************************************************************ +// +// Kokkos +// Manycore Performance-Portable Multidimensional Arrays +// +// Copyright (2012) Sandia Corporation +// +// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +// the U.S. Government retains certain rights in this software. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// 1. Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// 3. Neither the name of the Corporation nor the names of the +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Questions? Contact H. Carter Edwards (hcedwar@sandia.gov) +// +// ************************************************************************ +//@HEADER +*/ + + + +#include +#include +#include +#include + +#ifndef KOKKOS_RANDOM_HPP +#define KOKKOS_RANDOM_HPP + +// These generators are based on Vigna, Sebastiano (2014). "An experimental exploration of Marsaglia's xorshift generators, scrambled" +// See: http://arxiv.org/abs/1402.6246 + +namespace Kokkos { + + /*Template functions to get equidistributed random numbers from a generator for a specific Scalar type + + template + struct rand{ + + //Max value returned by draw(Generator& gen) + KOKKOS_INLINE_FUNCTION + static Scalar max(); + + //Returns a value between zero and max() + KOKKOS_INLINE_FUNCTION + static Scalar draw(Generator& gen); + + //Returns a value between zero and range() + //Note: for floating point values range can be larger than max() + KOKKOS_INLINE_FUNCTION + static Scalar draw(Generator& gen, const Scalar& range){} + + //Return value between start and end + KOKKOS_INLINE_FUNCTION + static Scalar draw(Generator& gen, const Scalar& start, const Scalar& end); + }; + + The Random number generators themselves have two components a state-pool and the actual generator + A state-pool manages a number of generators, so that each active thread is able to grep its own. + This allows the generation of random numbers which are independent between threads. Note that + in contrast to CuRand none of the functions of the pool (or the generator) are collectives, + i.e. all functions can be called inside conditionals. + + template + class Pool { + public: + //The Kokkos device type + typedef Device device_type; + //The actual generator type + typedef Generator generator_type; + + //Default constructor: does not initialize a pool + Pool(); + + //Initializing constructor: calls init(seed,Device_Specific_Number); + Pool(unsigned int seed); + + //Intialize Pool with seed as a starting seed with a pool_size of num_states + //The Random_XorShift64 generator is used in serial to initialize all states, + //thus the intialization process is platform independent and deterministic. + void init(unsigned int seed, int num_states); + + //Get a generator. This will lock one of the states, guaranteeing that each thread + //will have its private generator. Note: on Cuda getting a state involves atomics, + //and is thus not deterministic! + generator_type get_state(); + + //Give a state back to the pool. This unlocks the state, and writes the modified + //state of the generator back to the pool. + void free_state(generator_type gen); + + } + + template + class Generator { + public: + //The Kokkos device type + typedef DeviceType device_type; + + //Max return values of respective [X]rand[S]() functions + enum {MAX_URAND = 0xffffffffU}; + enum {MAX_URAND64 = 0xffffffffffffffffULL-1}; + enum {MAX_RAND = static_cast(0xffffffffU/2)}; + enum {MAX_RAND64 = static_cast(0xffffffffffffffffULL/2-1)}; + + + //Init with a state and the idx with respect to pool. Note: in serial the + //Generator can be used by just giving it the necessary state arguments + KOKKOS_INLINE_FUNCTION + Generator (STATE_ARGUMENTS, int state_idx = 0); + + //Draw a equidistributed uint32_t in the range (0,MAX_URAND] + KOKKOS_INLINE_FUNCTION + uint32_t urand(); + + //Draw a equidistributed uint64_t in the range (0,MAX_URAND64] + KOKKOS_INLINE_FUNCTION + uint64_t urand64(); + + //Draw a equidistributed uint32_t in the range (0,range] + KOKKOS_INLINE_FUNCTION + uint32_t urand(const uint32_t& range); + + //Draw a equidistributed uint32_t in the range (start,end] + KOKKOS_INLINE_FUNCTION + uint32_t urand(const uint32_t& start, const uint32_t& end ); + + //Draw a equidistributed uint64_t in the range (0,range] + KOKKOS_INLINE_FUNCTION + uint64_t urand64(const uint64_t& range); + + //Draw a equidistributed uint64_t in the range (start,end] + KOKKOS_INLINE_FUNCTION + uint64_t urand64(const uint64_t& start, const uint64_t& end ); + + //Draw a equidistributed int in the range (0,MAX_RAND] + KOKKOS_INLINE_FUNCTION + int rand(); + + //Draw a equidistributed int in the range (0,range] + KOKKOS_INLINE_FUNCTION + int rand(const int& range); + + //Draw a equidistributed int in the range (start,end] + KOKKOS_INLINE_FUNCTION + int rand(const int& start, const int& end ); + + //Draw a equidistributed int64_t in the range (0,MAX_RAND64] + KOKKOS_INLINE_FUNCTION + int64_t rand64(); + + //Draw a equidistributed int64_t in the range (0,range] + KOKKOS_INLINE_FUNCTION + int64_t rand64(const int64_t& range); + + //Draw a equidistributed int64_t in the range (start,end] + KOKKOS_INLINE_FUNCTION + int64_t rand64(const int64_t& start, const int64_t& end ); + + //Draw a equidistributed float in the range (0,1.0] + KOKKOS_INLINE_FUNCTION + float frand(); + + //Draw a equidistributed float in the range (0,range] + KOKKOS_INLINE_FUNCTION + float frand(const float& range); + + //Draw a equidistributed float in the range (start,end] + KOKKOS_INLINE_FUNCTION + float frand(const float& start, const float& end ); + + //Draw a equidistributed double in the range (0,1.0] + KOKKOS_INLINE_FUNCTION + double drand(); + + //Draw a equidistributed double in the range (0,range] + KOKKOS_INLINE_FUNCTION + double drand(const double& range); + + //Draw a equidistributed double in the range (start,end] + KOKKOS_INLINE_FUNCTION + double drand(const double& start, const double& end ); + + //Draw a standard normal distributed double + KOKKOS_INLINE_FUNCTION + double normal() ; + + //Draw a normal distributed double with given mean and standard deviation + KOKKOS_INLINE_FUNCTION + double normal(const double& mean, const double& std_dev=1.0); + } + + //Additional Functions: + + //Fills view with random numbers in the range (0,range] + template + void fill_random(ViewType view, PoolType pool, ViewType::value_type range); + + //Fills view with random numbers in the range (start,end] + template + void fill_random(ViewType view, PoolType pool, + ViewType::value_type start, ViewType::value_type end); + +*/ + + template + struct rand; + + + template + struct rand { + + KOKKOS_INLINE_FUNCTION + static short max(){return 127;} + KOKKOS_INLINE_FUNCTION + static short draw(Generator& gen) + {return short((gen.rand()&0xff+256)%256);} + KOKKOS_INLINE_FUNCTION + static short draw(Generator& gen, const char& range) + {return char(gen.rand(range));} + KOKKOS_INLINE_FUNCTION + static short draw(Generator& gen, const char& start, const char& end) + {return char(gen.rand(start,end));} + + }; + + template + struct rand { + KOKKOS_INLINE_FUNCTION + static short max(){return 32767;} + KOKKOS_INLINE_FUNCTION + static short draw(Generator& gen) + {return short((gen.rand()&0xffff+65536)%32768);} + KOKKOS_INLINE_FUNCTION + static short draw(Generator& gen, const short& range) + {return short(gen.rand(range));} + KOKKOS_INLINE_FUNCTION + static short draw(Generator& gen, const short& start, const short& end) + {return short(gen.rand(start,end));} + + }; + + template + struct rand { + KOKKOS_INLINE_FUNCTION + static int max(){return Generator::MAX_RAND;} + KOKKOS_INLINE_FUNCTION + static int draw(Generator& gen) + {return gen.rand();} + KOKKOS_INLINE_FUNCTION + static int draw(Generator& gen, const int& range) + {return gen.rand(range);} + KOKKOS_INLINE_FUNCTION + static int draw(Generator& gen, const int& start, const int& end) + {return gen.rand(start,end);} + + }; + + template + struct rand { + KOKKOS_INLINE_FUNCTION + static unsigned int max () { + return Generator::MAX_URAND; + } + KOKKOS_INLINE_FUNCTION + static unsigned int draw (Generator& gen) { + return gen.urand (); + } + KOKKOS_INLINE_FUNCTION + static unsigned int draw(Generator& gen, const unsigned int& range) { + return gen.urand (range); + } + KOKKOS_INLINE_FUNCTION + static unsigned int + draw (Generator& gen, const unsigned int& start, const unsigned int& end) { + return gen.urand (start, end); + } + }; + + template + struct rand { + KOKKOS_INLINE_FUNCTION + static long max () { + // FIXME (mfh 26 Oct 2014) It would be better to select the + // return value at compile time, using something like enable_if. + return sizeof (long) == 4 ? + static_cast (Generator::MAX_RAND) : + static_cast (Generator::MAX_RAND64); + } + KOKKOS_INLINE_FUNCTION + static long draw (Generator& gen) { + // FIXME (mfh 26 Oct 2014) It would be better to select the + // return value at compile time, using something like enable_if. + return sizeof (long) == 4 ? + static_cast (gen.rand ()) : + static_cast (gen.rand64 ()); + } + KOKKOS_INLINE_FUNCTION + static long draw (Generator& gen, const long& range) { + // FIXME (mfh 26 Oct 2014) It would be better to select the + // return value at compile time, using something like enable_if. + return sizeof (long) == 4 ? + static_cast (gen.rand (static_cast (range))) : + static_cast (gen.rand64 (range)); + } + KOKKOS_INLINE_FUNCTION + static long draw (Generator& gen, const long& start, const long& end) { + // FIXME (mfh 26 Oct 2014) It would be better to select the + // return value at compile time, using something like enable_if. + return sizeof (long) == 4 ? + static_cast (gen.rand (static_cast (start), + static_cast (end))) : + static_cast (gen.rand64 (start, end)); + } + }; + + template + struct rand { + KOKKOS_INLINE_FUNCTION + static unsigned long max () { + // FIXME (mfh 26 Oct 2014) It would be better to select the + // return value at compile time, using something like enable_if. + return sizeof (unsigned long) == 4 ? + static_cast (Generator::MAX_URAND) : + static_cast (Generator::MAX_URAND64); + } + KOKKOS_INLINE_FUNCTION + static unsigned long draw (Generator& gen) { + // FIXME (mfh 26 Oct 2014) It would be better to select the + // return value at compile time, using something like enable_if. + return sizeof (unsigned long) == 4 ? + static_cast (gen.urand ()) : + static_cast (gen.urand64 ()); + } + KOKKOS_INLINE_FUNCTION + static unsigned long draw(Generator& gen, const unsigned long& range) { + // FIXME (mfh 26 Oct 2014) It would be better to select the + // return value at compile time, using something like enable_if. + return sizeof (unsigned long) == 4 ? + static_cast (gen.urand (static_cast (range))) : + static_cast (gen.urand64 (range)); + } + KOKKOS_INLINE_FUNCTION + static unsigned long + draw (Generator& gen, const unsigned long& start, const unsigned long& end) { + // FIXME (mfh 26 Oct 2014) It would be better to select the + // return value at compile time, using something like enable_if. + return sizeof (unsigned long) == 4 ? + static_cast (gen.urand (static_cast (start), + static_cast (end))) : + static_cast (gen.urand64 (start, end)); + } + }; + + // NOTE (mfh 26 oct 2014) This is a partial specialization for long + // long, a C99 / C++11 signed type which is guaranteed to be at + // least 64 bits. Do NOT write a partial specialization for + // int64_t!!! This is just a typedef! It could be either long or + // long long. We don't know which a priori, and I've seen both. + // The types long and long long are guaranteed to differ, so it's + // always safe to specialize for both. + template + struct rand { + KOKKOS_INLINE_FUNCTION + static long long max () { + // FIXME (mfh 26 Oct 2014) It's legal for long long to be > 64 bits. + return Generator::MAX_RAND64; + } + KOKKOS_INLINE_FUNCTION + static long long draw (Generator& gen) { + // FIXME (mfh 26 Oct 2014) It's legal for long long to be > 64 bits. + return gen.rand64 (); + } + KOKKOS_INLINE_FUNCTION + static long long draw (Generator& gen, const long long& range) { + // FIXME (mfh 26 Oct 2014) It's legal for long long to be > 64 bits. + return gen.rand64 (range); + } + KOKKOS_INLINE_FUNCTION + static long long draw (Generator& gen, const long long& start, const long long& end) { + // FIXME (mfh 26 Oct 2014) It's legal for long long to be > 64 bits. + return gen.rand64 (start, end); + } + }; + + // NOTE (mfh 26 oct 2014) This is a partial specialization for + // unsigned long long, a C99 / C++11 unsigned type which is + // guaranteed to be at least 64 bits. Do NOT write a partial + // specialization for uint64_t!!! This is just a typedef! It could + // be either unsigned long or unsigned long long. We don't know + // which a priori, and I've seen both. The types unsigned long and + // unsigned long long are guaranteed to differ, so it's always safe + // to specialize for both. + template + struct rand { + KOKKOS_INLINE_FUNCTION + static unsigned long long max () { + // FIXME (mfh 26 Oct 2014) It's legal for unsigned long long to be > 64 bits. + return Generator::MAX_URAND64; + } + KOKKOS_INLINE_FUNCTION + static unsigned long long draw (Generator& gen) { + // FIXME (mfh 26 Oct 2014) It's legal for unsigned long long to be > 64 bits. + return gen.urand64 (); + } + KOKKOS_INLINE_FUNCTION + static unsigned long long draw (Generator& gen, const unsigned long long& range) { + // FIXME (mfh 26 Oct 2014) It's legal for long long to be > 64 bits. + return gen.urand64 (range); + } + KOKKOS_INLINE_FUNCTION + static unsigned long long + draw (Generator& gen, const unsigned long long& start, const unsigned long long& end) { + // FIXME (mfh 26 Oct 2014) It's legal for long long to be > 64 bits. + return gen.urand64 (start, end); + } + }; + + template + struct rand { + KOKKOS_INLINE_FUNCTION + static float max(){return 1.0f;} + KOKKOS_INLINE_FUNCTION + static float draw(Generator& gen) + {return gen.frand();} + KOKKOS_INLINE_FUNCTION + static float draw(Generator& gen, const float& range) + {return gen.frand(range);} + KOKKOS_INLINE_FUNCTION + static float draw(Generator& gen, const float& start, const float& end) + {return gen.frand(start,end);} + + }; + + template + struct rand { + KOKKOS_INLINE_FUNCTION + static double max(){return 1.0;} + KOKKOS_INLINE_FUNCTION + static double draw(Generator& gen) + {return gen.drand();} + KOKKOS_INLINE_FUNCTION + static double draw(Generator& gen, const double& range) + {return gen.drand(range);} + KOKKOS_INLINE_FUNCTION + static double draw(Generator& gen, const double& start, const double& end) + {return gen.drand(start,end);} + + }; + + template + class Random_XorShift64_Pool; + + template + class Random_XorShift64 { + private: + uint64_t state_; + const int state_idx_; + friend class Random_XorShift64_Pool; + public: + + typedef DeviceType device_type; + + enum {MAX_URAND = 0xffffffffU}; + enum {MAX_URAND64 = 0xffffffffffffffffULL-1}; + enum {MAX_RAND = static_cast(0xffffffff/2)}; + enum {MAX_RAND64 = static_cast(0xffffffffffffffffLL/2-1)}; + + KOKKOS_INLINE_FUNCTION + Random_XorShift64 (uint64_t state, int state_idx = 0) + : state_(state),state_idx_(state_idx){} + + KOKKOS_INLINE_FUNCTION + uint32_t urand() { + state_ ^= state_ >> 12; + state_ ^= state_ << 25; + state_ ^= state_ >> 27; + + uint64_t tmp = state_ * 2685821657736338717ULL; + tmp = tmp>>16; + return static_cast(tmp&MAX_URAND); + } + + KOKKOS_INLINE_FUNCTION + uint64_t urand64() { + state_ ^= state_ >> 12; + state_ ^= state_ << 25; + state_ ^= state_ >> 27; + return (state_ * 2685821657736338717ULL) - 1; + } + + KOKKOS_INLINE_FUNCTION + uint32_t urand(const uint32_t& range) { + const uint32_t max_val = (MAX_URAND/range)*range; + uint32_t tmp = urand(); + while(tmp>=max_val) + tmp = urand(); + return tmp%range; + } + + KOKKOS_INLINE_FUNCTION + uint32_t urand(const uint32_t& start, const uint32_t& end ) { + return urand(end-start)+start; + } + + KOKKOS_INLINE_FUNCTION + uint64_t urand64(const uint64_t& range) { + const uint64_t max_val = (MAX_URAND64/range)*range; + uint64_t tmp = urand64(); + while(tmp>=max_val) + tmp = urand64(); + return tmp%range; + } + + KOKKOS_INLINE_FUNCTION + uint64_t urand64(const uint64_t& start, const uint64_t& end ) { + return urand64(end-start)+start; + } + + KOKKOS_INLINE_FUNCTION + int rand() { + return static_cast(urand()/2); + } + + KOKKOS_INLINE_FUNCTION + int rand(const int& range) { + const int max_val = (MAX_RAND/range)*range; + int tmp = rand(); + while(tmp>=max_val) + tmp = rand(); + return tmp%range; + } + + KOKKOS_INLINE_FUNCTION + int rand(const int& start, const int& end ) { + return rand(end-start)+start; + } + + KOKKOS_INLINE_FUNCTION + int64_t rand64() { + return static_cast(urand64()/2); + } + + KOKKOS_INLINE_FUNCTION + int64_t rand64(const int64_t& range) { + const int64_t max_val = (MAX_RAND64/range)*range; + int64_t tmp = rand64(); + while(tmp>=max_val) + tmp = rand64(); + return tmp%range; + } + + KOKKOS_INLINE_FUNCTION + int64_t rand64(const int64_t& start, const int64_t& end ) { + return rand64(end-start)+start; + } + + KOKKOS_INLINE_FUNCTION + float frand() { + return 1.0f * urand64()/MAX_URAND64; + } + + KOKKOS_INLINE_FUNCTION + float frand(const float& range) { + return range * urand64()/MAX_URAND64; + } + + KOKKOS_INLINE_FUNCTION + float frand(const float& start, const float& end ) { + return frand(end-start)+start; + } + + KOKKOS_INLINE_FUNCTION + double drand() { + return 1.0 * urand64()/MAX_URAND64; + } + + KOKKOS_INLINE_FUNCTION + double drand(const double& range) { + return range * urand64()/MAX_URAND64; + } + + KOKKOS_INLINE_FUNCTION + double drand(const double& start, const double& end ) { + return drand(end-start)+start; + } + + //Marsaglia polar method for drawing a standard normal distributed random number + KOKKOS_INLINE_FUNCTION + double normal() { + double S = 2.0; + double U; + while(S>=1.0) { + U = drand(); + const double V = drand(); + S = U*U+V*V; + } + return U*sqrt(-2.0*log(S)/S); + } + + KOKKOS_INLINE_FUNCTION + double normal(const double& mean, const double& std_dev=1.0) { + return mean + normal()*std_dev; + } + + }; + + template + class Random_XorShift64_Pool { + private: + typedef View lock_type; + typedef View state_data_type; + lock_type locks_; + state_data_type state_; + int num_states_; + + public: + typedef Random_XorShift64 generator_type; + typedef DeviceType device_type; + + Random_XorShift64_Pool() { + num_states_ = 0; + } + Random_XorShift64_Pool(unsigned int seed) { + num_states_ = 0; + init(seed,DeviceType::max_hardware_threads()); + } + + Random_XorShift64_Pool(const Random_XorShift64_Pool& src): + locks_(src.locks_), + state_(src.state_), + num_states_(src.num_states_) + {} + + Random_XorShift64_Pool operator = (const Random_XorShift64_Pool& src) { + locks_ = src.locks_; + state_ = src.state_; + num_states_ = src.num_states_; + return *this; + } + + void init(unsigned int seed, int num_states) { + num_states_ = num_states; + + locks_ = lock_type("Kokkos::Random_XorShift64::locks",num_states_); + state_ = state_data_type("Kokkos::Random_XorShift64::state",num_states_); + + typename state_data_type::HostMirror h_state = create_mirror_view(state_); + typename lock_type::HostMirror h_lock = create_mirror_view(locks_); + + // Execute on the HostMirror's default execution space. + Random_XorShift64 gen(seed,0); + for(int i = 0; i < 17; i++) + gen.rand(); + for(int i = 0; i < num_states_; i++) { + int n1 = gen.rand(); + int n2 = gen.rand(); + int n3 = gen.rand(); + int n4 = gen.rand(); + h_state(i) = (((static_cast(n1)) & 0xffff)<<00) | + (((static_cast(n2)) & 0xffff)<<16) | + (((static_cast(n3)) & 0xffff)<<32) | + (((static_cast(n4)) & 0xffff)<<48); + h_lock(i) = 0; + } + deep_copy(state_,h_state); + deep_copy(locks_,h_lock); + } + + KOKKOS_INLINE_FUNCTION + Random_XorShift64 get_state() const { + const int i = DeviceType::hardware_thread_id();; + return Random_XorShift64(state_(i),i); + } + + KOKKOS_INLINE_FUNCTION + void free_state(const Random_XorShift64& state) const { + state_(state.state_idx_) = state.state_; + } + }; + + + template + class Random_XorShift1024_Pool; + + template + class Random_XorShift1024 { + private: + int p_; + const int state_idx_; + uint64_t state_[16]; + friend class Random_XorShift1024_Pool; + public: + + typedef DeviceType device_type; + + enum {MAX_URAND = 0xffffffffU}; + enum {MAX_URAND64 = 0xffffffffffffffffULL-1}; + enum {MAX_RAND = static_cast(0xffffffffU/2)}; + enum {MAX_RAND64 = static_cast(0xffffffffffffffffULL/2-1)}; + + KOKKOS_INLINE_FUNCTION + Random_XorShift1024 (uint64_t* state, int p, int state_idx = 0): + p_(p),state_idx_(state_idx){ + for(int i=0 ; i<16; i++) + state_[i] = state[i]; + } + + KOKKOS_INLINE_FUNCTION + uint32_t urand() { + uint64_t state_0 = state_[ p_ ]; + uint64_t state_1 = state_[ p_ = ( p_ + 1 ) & 15 ]; + state_1 ^= state_1 << 31; + state_1 ^= state_1 >> 11; + state_0 ^= state_0 >> 30; + uint64_t tmp = ( state_[ p_ ] = state_0 ^ state_1 ) * 1181783497276652981ULL; + tmp = tmp>>16; + return static_cast(tmp&MAX_URAND); + } + + KOKKOS_INLINE_FUNCTION + uint64_t urand64() { + uint64_t state_0 = state_[ p_ ]; + uint64_t state_1 = state_[ p_ = ( p_ + 1 ) & 15 ]; + state_1 ^= state_1 << 31; + state_1 ^= state_1 >> 11; + state_0 ^= state_0 >> 30; + return (( state_[ p_ ] = state_0 ^ state_1 ) * 1181783497276652981LL) - 1; + } + + KOKKOS_INLINE_FUNCTION + uint32_t urand(const uint32_t& range) { + const uint32_t max_val = (MAX_URAND/range)*range; + uint32_t tmp = urand(); + while(tmp>=max_val) + tmp = urand(); + return tmp%range; + } + + KOKKOS_INLINE_FUNCTION + uint32_t urand(const uint32_t& start, const uint32_t& end ) { + return urand(end-start)+start; + } + + KOKKOS_INLINE_FUNCTION + uint64_t urand64(const uint64_t& range) { + const uint64_t max_val = (MAX_URAND64/range)*range; + uint64_t tmp = urand64(); + while(tmp>=max_val) + tmp = urand64(); + return tmp%range; + } + + KOKKOS_INLINE_FUNCTION + uint64_t urand64(const uint64_t& start, const uint64_t& end ) { + return urand64(end-start)+start; + } + + KOKKOS_INLINE_FUNCTION + int rand() { + return static_cast(urand()/2); + } + + KOKKOS_INLINE_FUNCTION + int rand(const int& range) { + const int max_val = (MAX_RAND/range)*range; + int tmp = rand(); + while(tmp>=max_val) + tmp = rand(); + return tmp%range; + } + + KOKKOS_INLINE_FUNCTION + int rand(const int& start, const int& end ) { + return rand(end-start)+start; + } + + KOKKOS_INLINE_FUNCTION + int64_t rand64() { + return static_cast(urand64()/2); + } + + KOKKOS_INLINE_FUNCTION + int64_t rand64(const int64_t& range) { + const int64_t max_val = (MAX_RAND64/range)*range; + int64_t tmp = rand64(); + while(tmp>=max_val) + tmp = rand64(); + return tmp%range; + } + + KOKKOS_INLINE_FUNCTION + int64_t rand64(const int64_t& start, const int64_t& end ) { + return rand64(end-start)+start; + } + + KOKKOS_INLINE_FUNCTION + float frand() { + return 1.0f * urand64()/MAX_URAND64; + } + + KOKKOS_INLINE_FUNCTION + float frand(const float& range) { + return range * urand64()/MAX_URAND64; + } + + KOKKOS_INLINE_FUNCTION + float frand(const float& start, const float& end ) { + return frand(end-start)+start; + } + + KOKKOS_INLINE_FUNCTION + double drand() { + return 1.0 * urand64()/MAX_URAND64; + } + + KOKKOS_INLINE_FUNCTION + double drand(const double& range) { + return range * urand64()/MAX_URAND64; + } + + KOKKOS_INLINE_FUNCTION + double drand(const double& start, const double& end ) { + return frand(end-start)+start; + } + + //Marsaglia polar method for drawing a standard normal distributed random number + KOKKOS_INLINE_FUNCTION + double normal() { + double S = 2.0; + double U; + while(S>=1.0) { + U = drand(); + const double V = drand(); + S = U*U+V*V; + } + return U*sqrt(-2.0*log(S)/S); + } + + KOKKOS_INLINE_FUNCTION + double normal(const double& mean, const double& std_dev=1.0) { + return mean + normal()*std_dev; + } + }; + + + template + class Random_XorShift1024_Pool { + private: + typedef View int_view_type; + typedef View state_data_type; + + int_view_type locks_; + state_data_type state_; + int_view_type p_; + int num_states_; + + public: + typedef Random_XorShift1024 generator_type; + + typedef DeviceType device_type; + + Random_XorShift1024_Pool() { + num_states_ = 0; + } + + inline + Random_XorShift1024_Pool(unsigned int seed){ + num_states_ = 0; + init(seed,DeviceType::max_hardware_threads()); + } + + Random_XorShift1024_Pool(const Random_XorShift1024_Pool& src): + locks_(src.locks_), + state_(src.state_), + p_(src.p_), + num_states_(src.num_states_) + {} + + Random_XorShift1024_Pool operator = (const Random_XorShift1024_Pool& src) { + locks_ = src.locks_; + state_ = src.state_; + p_ = src.p_; + num_states_ = src.num_states_; + return *this; + } + + inline + void init(unsigned int seed, int num_states) { + num_states_ = num_states; + + locks_ = int_view_type("Kokkos::Random_XorShift1024::locks",num_states_); + state_ = state_data_type("Kokkos::Random_XorShift1024::state",num_states_); + p_ = int_view_type("Kokkos::Random_XorShift1024::p",num_states_); + + typename state_data_type::HostMirror h_state = create_mirror_view(state_); + typename int_view_type::HostMirror h_lock = create_mirror_view(locks_); + typename int_view_type::HostMirror h_p = create_mirror_view(p_); + + // Execute on the HostMirror's default execution space. + Random_XorShift64 gen(seed,0); + for(int i = 0; i < 17; i++) + gen.rand(); + for(int i = 0; i < num_states_; i++) { + for(int j = 0; j < 16 ; j++) { + int n1 = gen.rand(); + int n2 = gen.rand(); + int n3 = gen.rand(); + int n4 = gen.rand(); + h_state(i,j) = (((static_cast(n1)) & 0xffff)<<00) | + (((static_cast(n2)) & 0xffff)<<16) | + (((static_cast(n3)) & 0xffff)<<32) | + (((static_cast(n4)) & 0xffff)<<48); + } + h_p(i) = 0; + h_lock(i) = 0; + } + deep_copy(state_,h_state); + deep_copy(locks_,h_lock); + } + + KOKKOS_INLINE_FUNCTION + Random_XorShift1024 get_state() const { + const int i = DeviceType::hardware_thread_id(); + return Random_XorShift1024(&state_(i,0),p_(i),i); + }; + + KOKKOS_INLINE_FUNCTION + void free_state(const Random_XorShift1024& state) const { + for(int i = 0; i<16; i++) + state_(state.state_idx_,i) = state.state_[i]; + p_(state.state_idx_) = state.p_; + } + }; + +#if defined(KOKKOS_HAVE_CUDA) && defined(__CUDACC__) + + template<> + class Random_XorShift1024 { + private: + int p_; + const int state_idx_; + uint64_t* state_; + friend class Random_XorShift1024_Pool; + public: + + typedef Kokkos::Cuda device_type; + + enum {MAX_URAND = 0xffffffffU}; + enum {MAX_URAND64 = 0xffffffffffffffffULL-1}; + enum {MAX_RAND = static_cast(0xffffffffU/2)}; + enum {MAX_RAND64 = static_cast(0xffffffffffffffffULL/2-1)}; + + KOKKOS_INLINE_FUNCTION + Random_XorShift1024 (uint64_t* state, int p, int state_idx = 0): + p_(p),state_idx_(state_idx),state_(state){ + } + + KOKKOS_INLINE_FUNCTION + uint32_t urand() { + uint64_t state_0 = state_[ p_ ]; + uint64_t state_1 = state_[ p_ = ( p_ + 1 ) & 15 ]; + state_1 ^= state_1 << 31; + state_1 ^= state_1 >> 11; + state_0 ^= state_0 >> 30; + uint64_t tmp = ( state_[ p_ ] = state_0 ^ state_1 ) * 1181783497276652981ULL; + tmp = tmp>>16; + return static_cast(tmp&MAX_URAND); + } + + KOKKOS_INLINE_FUNCTION + uint64_t urand64() { + uint64_t state_0 = state_[ p_ ]; + uint64_t state_1 = state_[ p_ = ( p_ + 1 ) & 15 ]; + state_1 ^= state_1 << 31; + state_1 ^= state_1 >> 11; + state_0 ^= state_0 >> 30; + return (( state_[ p_ ] = state_0 ^ state_1 ) * 1181783497276652981LL) - 1; + } + + KOKKOS_INLINE_FUNCTION + uint32_t urand(const uint32_t& range) { + const uint32_t max_val = (MAX_URAND/range)*range; + uint32_t tmp = urand(); + while(tmp>=max_val) + urand(); + return tmp%range; + } + + KOKKOS_INLINE_FUNCTION + uint32_t urand(const uint32_t& start, const uint32_t& end ) { + return urand(end-start)+start; + } + + KOKKOS_INLINE_FUNCTION + uint64_t urand64(const uint64_t& range) { + const uint64_t max_val = (MAX_URAND64/range)*range; + uint64_t tmp = urand64(); + while(tmp>=max_val) + urand64(); + return tmp%range; + } + + KOKKOS_INLINE_FUNCTION + uint64_t urand64(const uint64_t& start, const uint64_t& end ) { + return urand64(end-start)+start; + } + + KOKKOS_INLINE_FUNCTION + int rand() { + return static_cast(urand()/2); + } + + KOKKOS_INLINE_FUNCTION + int rand(const int& range) { + const int max_val = (MAX_RAND/range)*range; + int tmp = rand(); + while(tmp>=max_val) + rand(); + return tmp%range; + } + + KOKKOS_INLINE_FUNCTION + int rand(const int& start, const int& end ) { + return rand(end-start)+start; + } + + KOKKOS_INLINE_FUNCTION + int64_t rand64() { + return static_cast(urand64()/2); + } + + KOKKOS_INLINE_FUNCTION + int64_t rand64(const int64_t& range) { + const int64_t max_val = (MAX_RAND64/range)*range; + int64_t tmp = rand64(); + while(tmp>=max_val) + rand64(); + return tmp%range; + } + + KOKKOS_INLINE_FUNCTION + int64_t rand64(const int64_t& start, const int64_t& end ) { + return rand64(end-start)+start; + } + + KOKKOS_INLINE_FUNCTION + float frand() { + return 1.0f * urand64()/MAX_URAND64; + } + + KOKKOS_INLINE_FUNCTION + float frand(const float& range) { + return range * urand64()/MAX_URAND64; + } + + KOKKOS_INLINE_FUNCTION + float frand(const float& start, const float& end ) { + return frand(end-start)+start; + } + + KOKKOS_INLINE_FUNCTION + double drand() { + return 1.0 * urand64()/MAX_URAND64; + } + + KOKKOS_INLINE_FUNCTION + double drand(const double& range) { + return range * urand64()/MAX_URAND64; + } + + KOKKOS_INLINE_FUNCTION + double drand(const double& start, const double& end ) { + return frand(end-start)+start; + } + + //Marsaglia polar method for drawing a standard normal distributed random number + KOKKOS_INLINE_FUNCTION + double normal() { + double S = 2.0; + double U; + while(S>=1.0) { + U = drand(); + const double V = drand(); + S = U*U+V*V; + } + return U*sqrt(-2.0*log(S)/S); + } + + KOKKOS_INLINE_FUNCTION + double normal(const double& mean, const double& std_dev=1.0) { + return mean + normal()*std_dev; + } + }; + +template<> +inline +Random_XorShift64_Pool::Random_XorShift64_Pool(unsigned int seed) { + num_states_ = 0; + init(seed,4*32768); +} + +template<> +KOKKOS_INLINE_FUNCTION +Random_XorShift64 Random_XorShift64_Pool::get_state() const { +#ifdef __CUDA_ARCH__ + const int i_offset = (threadIdx.x*blockDim.y + threadIdx.y)*blockDim.z+threadIdx.z; + int i = ((blockIdx.x*gridDim.y+blockIdx.y)*gridDim.z + blockIdx.z) * + blockDim.x*blockDim.y*blockDim.z + i_offset; + while(Kokkos::atomic_compare_exchange(&locks_(i),0,1)) { + i+=blockDim.x*blockDim.y*blockDim.z; + if(i>=num_states_) {i = i_offset;} + } + + return Random_XorShift64(state_(i),i); +#else + return Random_XorShift64(state_(0),0); +#endif +} + +template<> +KOKKOS_INLINE_FUNCTION +void Random_XorShift64_Pool::free_state(const Random_XorShift64 &state) const { +#ifdef __CUDA_ARCH__ + state_(state.state_idx_) = state.state_; + locks_(state.state_idx_) = 0; + return; +#endif +} + + +template<> +inline +Random_XorShift1024_Pool::Random_XorShift1024_Pool(unsigned int seed) { + num_states_ = 0; + init(seed,4*32768); +} + +template<> +KOKKOS_INLINE_FUNCTION +Random_XorShift1024 Random_XorShift1024_Pool::get_state() const { +#ifdef __CUDA_ARCH__ + const int i_offset = (threadIdx.x*blockDim.y + threadIdx.y)*blockDim.z+threadIdx.z; + int i = ((blockIdx.x*gridDim.y+blockIdx.y)*gridDim.z + blockIdx.z) * + blockDim.x*blockDim.y*blockDim.z + i_offset; + while(Kokkos::atomic_compare_exchange(&locks_(i),0,1)) { + i+=blockDim.x*blockDim.y*blockDim.z; + if(i>=num_states_) {i = i_offset;} + } + + return Random_XorShift1024(&state_(i,0), p_(i), i); +#else + return Random_XorShift1024(&state_(0,0), p_(0), 0); +#endif +} + +template<> +KOKKOS_INLINE_FUNCTION +void Random_XorShift1024_Pool::free_state(const Random_XorShift1024 &state) const { +#ifdef __CUDA_ARCH__ + for(int i=0; i<16; i++) + state_(state.state_idx_,i) = state.state_[i]; + locks_(state.state_idx_) = 0; + return; +#endif +} + + +#endif + + + +template +struct fill_random_functor_range; +template +struct fill_random_functor_begin_end; + +template +struct fill_random_functor_range{ + typedef typename ViewType::device_type device_type; + ViewType a; + RandomPool rand_pool; + typename ViewType::const_value_type range; + + typedef rand Rand; + + fill_random_functor_range(ViewType a_, RandomPool rand_pool_, + typename ViewType::const_value_type range_): + a(a_),rand_pool(rand_pool_),range(range_) {} + + KOKKOS_INLINE_FUNCTION + void operator() (unsigned int i) const { + typename RandomPool::generator_type gen = rand_pool.get_state(); + for(unsigned int j=0;j +struct fill_random_functor_range{ + typedef typename ViewType::device_type device_type; + ViewType a; + RandomPool rand_pool; + typename ViewType::const_value_type range; + + typedef rand Rand; + + fill_random_functor_range(ViewType a_, RandomPool rand_pool_, + typename ViewType::const_value_type range_): + a(a_),rand_pool(rand_pool_),range(range_) {} + + KOKKOS_INLINE_FUNCTION + void operator() (unsigned int i) const { + typename RandomPool::generator_type gen = rand_pool.get_state(); + for(unsigned int j=0;j +struct fill_random_functor_range{ + typedef typename ViewType::device_type device_type; + ViewType a; + RandomPool rand_pool; + typename ViewType::const_value_type range; + + typedef rand Rand; + + fill_random_functor_range(ViewType a_, RandomPool rand_pool_, + typename ViewType::const_value_type range_): + a(a_),rand_pool(rand_pool_),range(range_) {} + + KOKKOS_INLINE_FUNCTION + void operator() (unsigned int i) const { + typename RandomPool::generator_type gen = rand_pool.get_state(); + for(unsigned int j=0;j +struct fill_random_functor_range{ + typedef typename ViewType::device_type device_type; + ViewType a; + RandomPool rand_pool; + typename ViewType::const_value_type range; + + typedef rand Rand; + + fill_random_functor_range(ViewType a_, RandomPool rand_pool_, + typename ViewType::const_value_type range_): + a(a_),rand_pool(rand_pool_),range(range_) {} + + KOKKOS_INLINE_FUNCTION + void operator() (unsigned int i) const { + typename RandomPool::generator_type gen = rand_pool.get_state(); + for(unsigned int j=0;j +struct fill_random_functor_range{ + typedef typename ViewType::device_type device_type; + ViewType a; + RandomPool rand_pool; + typename ViewType::const_value_type range; + + typedef rand Rand; + + fill_random_functor_range(ViewType a_, RandomPool rand_pool_, + typename ViewType::const_value_type range_): + a(a_),rand_pool(rand_pool_),range(range_) {} + + KOKKOS_INLINE_FUNCTION + void operator() (unsigned int i) const { + typename RandomPool::generator_type gen = rand_pool.get_state(); + for(unsigned int j=0;j +struct fill_random_functor_range{ + typedef typename ViewType::device_type device_type; + ViewType a; + RandomPool rand_pool; + typename ViewType::const_value_type range; + + typedef rand Rand; + + fill_random_functor_range(ViewType a_, RandomPool rand_pool_, + typename ViewType::const_value_type range_): + a(a_),rand_pool(rand_pool_),range(range_) {} + + KOKKOS_INLINE_FUNCTION + void operator() (unsigned int i) const { + typename RandomPool::generator_type gen = rand_pool.get_state(); + for(unsigned int j=0;j +struct fill_random_functor_range{ + typedef typename ViewType::device_type device_type; + ViewType a; + RandomPool rand_pool; + typename ViewType::const_value_type range; + + typedef rand Rand; + + fill_random_functor_range(ViewType a_, RandomPool rand_pool_, + typename ViewType::const_value_type range_): + a(a_),rand_pool(rand_pool_),range(range_) {} + + KOKKOS_INLINE_FUNCTION + void operator() (unsigned int i) const { + typename RandomPool::generator_type gen = rand_pool.get_state(); + for(unsigned int j=0;j +struct fill_random_functor_range{ + typedef typename ViewType::device_type device_type; + ViewType a; + RandomPool rand_pool; + typename ViewType::const_value_type range; + + typedef rand Rand; + + fill_random_functor_range(ViewType a_, RandomPool rand_pool_, + typename ViewType::const_value_type range_): + a(a_),rand_pool(rand_pool_),range(range_) {} + + KOKKOS_INLINE_FUNCTION + void operator() (unsigned int i) const { + typename RandomPool::generator_type gen = rand_pool.get_state(); + for(unsigned int j=0;j +struct fill_random_functor_begin_end{ + typedef typename ViewType::device_type device_type; + ViewType a; + RandomPool rand_pool; + typename ViewType::const_value_type begin,end; + + typedef rand Rand; + + fill_random_functor_begin_end(ViewType a_, RandomPool rand_pool_, + typename ViewType::const_value_type begin_, typename ViewType::const_value_type end_): + a(a_),rand_pool(rand_pool_),begin(begin_),end(end_) {} + + KOKKOS_INLINE_FUNCTION + void operator() (unsigned int i) const { + typename RandomPool::generator_type gen = rand_pool.get_state(); + for(unsigned int j=0;j +struct fill_random_functor_begin_end{ + typedef typename ViewType::device_type device_type; + ViewType a; + RandomPool rand_pool; + typename ViewType::const_value_type begin,end; + + typedef rand Rand; + + fill_random_functor_begin_end(ViewType a_, RandomPool rand_pool_, + typename ViewType::const_value_type begin_, typename ViewType::const_value_type end_): + a(a_),rand_pool(rand_pool_),begin(begin_),end(end_) {} + + KOKKOS_INLINE_FUNCTION + void operator() (unsigned int i) const { + typename RandomPool::generator_type gen = rand_pool.get_state(); + for(unsigned int j=0;j +struct fill_random_functor_begin_end{ + typedef typename ViewType::device_type device_type; + ViewType a; + RandomPool rand_pool; + typename ViewType::const_value_type begin,end; + + typedef rand Rand; + + fill_random_functor_begin_end(ViewType a_, RandomPool rand_pool_, + typename ViewType::const_value_type begin_, typename ViewType::const_value_type end_): + a(a_),rand_pool(rand_pool_),begin(begin_),end(end_) {} + + KOKKOS_INLINE_FUNCTION + void operator() (unsigned int i) const { + typename RandomPool::generator_type gen = rand_pool.get_state(); + for(unsigned int j=0;j +struct fill_random_functor_begin_end{ + typedef typename ViewType::device_type device_type; + ViewType a; + RandomPool rand_pool; + typename ViewType::const_value_type begin,end; + + typedef rand Rand; + + fill_random_functor_begin_end(ViewType a_, RandomPool rand_pool_, + typename ViewType::const_value_type begin_, typename ViewType::const_value_type end_): + a(a_),rand_pool(rand_pool_),begin(begin_),end(end_) {} + + KOKKOS_INLINE_FUNCTION + void operator() (unsigned int i) const { + typename RandomPool::generator_type gen = rand_pool.get_state(); + for(unsigned int j=0;j +struct fill_random_functor_begin_end{ + typedef typename ViewType::device_type device_type; + ViewType a; + RandomPool rand_pool; + typename ViewType::const_value_type begin,end; + + typedef rand Rand; + + fill_random_functor_begin_end(ViewType a_, RandomPool rand_pool_, + typename ViewType::const_value_type begin_, typename ViewType::const_value_type end_): + a(a_),rand_pool(rand_pool_),begin(begin_),end(end_) {} + + KOKKOS_INLINE_FUNCTION + void operator() (unsigned int i) const { + typename RandomPool::generator_type gen = rand_pool.get_state(); + for(unsigned int j=0;j +struct fill_random_functor_begin_end{ + typedef typename ViewType::device_type device_type; + ViewType a; + RandomPool rand_pool; + typename ViewType::const_value_type begin,end; + + typedef rand Rand; + + fill_random_functor_begin_end(ViewType a_, RandomPool rand_pool_, + typename ViewType::const_value_type begin_, typename ViewType::const_value_type end_): + a(a_),rand_pool(rand_pool_),begin(begin_),end(end_) {} + + KOKKOS_INLINE_FUNCTION + void operator() (unsigned int i) const { + typename RandomPool::generator_type gen = rand_pool.get_state(); + for(unsigned int j=0;j +struct fill_random_functor_begin_end{ + typedef typename ViewType::device_type device_type; + ViewType a; + RandomPool rand_pool; + typename ViewType::const_value_type begin,end; + + typedef rand Rand; + + fill_random_functor_begin_end(ViewType a_, RandomPool rand_pool_, + typename ViewType::const_value_type begin_, typename ViewType::const_value_type end_): + a(a_),rand_pool(rand_pool_),begin(begin_),end(end_) {} + + KOKKOS_INLINE_FUNCTION + void operator() (unsigned int i) const { + typename RandomPool::generator_type gen = rand_pool.get_state(); + for(unsigned int j=0;j +struct fill_random_functor_begin_end{ + typedef typename ViewType::device_type device_type; + ViewType a; + RandomPool rand_pool; + typename ViewType::const_value_type begin,end; + + typedef rand Rand; + + fill_random_functor_begin_end(ViewType a_, RandomPool rand_pool_, + typename ViewType::const_value_type begin_, typename ViewType::const_value_type end_): + a(a_),rand_pool(rand_pool_),begin(begin_),end(end_) {} + + KOKKOS_INLINE_FUNCTION + void operator() (unsigned int i) const { + typename RandomPool::generator_type gen = rand_pool.get_state(); + for(unsigned int j=0;j +void fill_random(ViewType a, RandomPool g, typename ViewType::const_value_type range) { + int64_t LDA = a.dimension_0(); + if(LDA>0) + parallel_for((LDA+127)/128,fill_random_functor_range(a,g,range)); +} + +template +void fill_random(ViewType a, RandomPool g, typename ViewType::const_value_type begin,typename ViewType::const_value_type end ) { + int64_t LDA = a.dimension_0(); + if(LDA>0) + parallel_for((LDA+127)/128,fill_random_functor_begin_end(a,g,begin,end)); +} +} + +#endif diff --git a/lib/kokkos/algorithms/src/Kokkos_Sort.hpp b/lib/kokkos/algorithms/src/Kokkos_Sort.hpp new file mode 100755 index 0000000000..99bd2ff12c --- /dev/null +++ b/lib/kokkos/algorithms/src/Kokkos_Sort.hpp @@ -0,0 +1,486 @@ +/* +//@HEADER +// ************************************************************************ +// +// Kokkos +// Manycore Performance-Portable Multidimensional Arrays +// +// Copyright (2012) Sandia Corporation +// +// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +// the U.S. Government retains certain rights in this software. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// 1. Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// 3. Neither the name of the Corporation nor the names of the +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Questions? Contact H. Carter Edwards (hcedwar@sandia.gov) +// +// ************************************************************************ +//@HEADER +*/ + + +#ifndef KOKKOS_SORT_HPP_ +#define KOKKOS_SORT_HPP_ + +#include + +#include + +namespace Kokkos { + + namespace SortImpl { + + template + struct CopyOp; + + template + struct CopyOp { + template + KOKKOS_INLINE_FUNCTION + static void copy(DstType& dst, size_t i_dst, + SrcType& src, size_t i_src ) { + dst(i_dst) = src(i_src); + } + }; + + template + struct CopyOp { + template + KOKKOS_INLINE_FUNCTION + static void copy(DstType& dst, size_t i_dst, + SrcType& src, size_t i_src ) { + for(int j = 0;j< (int) dst.dimension_1(); j++) + dst(i_dst,j) = src(i_src,j); + } + }; + + template + struct CopyOp { + template + KOKKOS_INLINE_FUNCTION + static void copy(DstType& dst, size_t i_dst, + SrcType& src, size_t i_src ) { + for(int j = 0; j +class BinSort { + + +public: + template + struct bin_sort_sort_functor { + typedef ExecutionSpace execution_space; + typedef typename ValuesViewType::non_const_type values_view_type; + typedef typename ValuesViewType::const_type const_values_view_type; + Kokkos::View > values; + values_view_type sorted_values; + typename PermuteViewType::const_type sort_order; + bin_sort_sort_functor(const_values_view_type values_, values_view_type sorted_values_, PermuteViewType sort_order_): + values(values_),sorted_values(sorted_values_),sort_order(sort_order_) {} + + KOKKOS_INLINE_FUNCTION + void operator() (const int& i) const { + //printf("Sort: %i %i\n",i,sort_order(i)); + CopyOp::copy(sorted_values,i,values,sort_order(i)); + } + }; + + typedef ExecutionSpace execution_space; + typedef BinSortOp bin_op_type; + + struct bin_count_tag {}; + struct bin_offset_tag {}; + struct bin_binning_tag {}; + struct bin_sort_bins_tag {}; + +public: + typedef SizeType size_type; + typedef size_type value_type; + + typedef Kokkos::View offset_type; + typedef Kokkos::View bin_count_type; + + + typedef Kokkos::View const_key_view_type; + typedef Kokkos::View > const_rnd_key_view_type; + + typedef typename KeyViewType::non_const_value_type non_const_key_scalar; + typedef typename KeyViewType::const_value_type const_key_scalar; + +private: + const_key_view_type keys; + const_rnd_key_view_type keys_rnd; + +public: + BinSortOp bin_op; + + offset_type bin_offsets; + + Kokkos::View > bin_count_atomic; + bin_count_type bin_count_const; + + offset_type sort_order; + + bool sort_within_bins; + +public: + + // Constructor: takes the keys, the binning_operator and optionally whether to sort within bins (default false) + BinSort(const_key_view_type keys_, BinSortOp bin_op_, + bool sort_within_bins_ = false) + :keys(keys_),keys_rnd(keys_), bin_op(bin_op_) { + + bin_count_atomic = Kokkos::View("Kokkos::SortImpl::BinSortFunctor::bin_count",bin_op.max_bins()); + bin_count_const = bin_count_atomic; + bin_offsets = offset_type("Kokkos::SortImpl::BinSortFunctor::bin_offsets",bin_op.max_bins()); + sort_order = offset_type("PermutationVector",keys.dimension_0()); + sort_within_bins = sort_within_bins_; + } + + // Create the permutation vector, the bin_offset array and the bin_count array. Can be called again if keys changed + void create_permute_vector() { + Kokkos::parallel_for (Kokkos::RangePolicy (0,keys.dimension_0()),*this); + Kokkos::parallel_scan(Kokkos::RangePolicy (0,bin_op.max_bins()) ,*this); + + Kokkos::deep_copy(bin_count_atomic,0); + Kokkos::parallel_for (Kokkos::RangePolicy (0,keys.dimension_0()),*this); + + if(sort_within_bins) + Kokkos::parallel_for (Kokkos::RangePolicy(0,bin_op.max_bins()) ,*this); + } + + // Sort a view with respect ot the first dimension using the permutation array + template + void sort(ValuesViewType values) { + ValuesViewType sorted_values = ValuesViewType("Copy", + values.dimension_0(), + values.dimension_1(), + values.dimension_2(), + values.dimension_3(), + values.dimension_4(), + values.dimension_5(), + values.dimension_6(), + values.dimension_7()); + + parallel_for(values.dimension_0(), + bin_sort_sort_functor >(values,sorted_values,sort_order)); + + deep_copy(values,sorted_values); + } + + // Get the permutation vector + KOKKOS_INLINE_FUNCTION + offset_type get_permute_vector() const { return sort_order;} + + // Get the start offsets for each bin + KOKKOS_INLINE_FUNCTION + offset_type get_bin_offsets() const { return bin_offsets;} + + // Get the count for each bin + KOKKOS_INLINE_FUNCTION + bin_count_type get_bin_count() const {return bin_count_const;} + +public: + KOKKOS_INLINE_FUNCTION + void operator() (const bin_count_tag& tag, const int& i) const { + bin_count_atomic(bin_op.bin(keys,i))++; + } + + KOKKOS_INLINE_FUNCTION + void operator() (const bin_offset_tag& tag, const int& i, value_type& offset, const bool& final) const { + if(final) { + bin_offsets(i) = offset; + } + offset+=bin_count_const(i); + } + + KOKKOS_INLINE_FUNCTION + void operator() (const bin_binning_tag& tag, const int& i) const { + const int bin = bin_op.bin(keys,i); + const int count = bin_count_atomic(bin)++; + + sort_order(bin_offsets(bin) + count) = i; + } + + KOKKOS_INLINE_FUNCTION + void operator() (const bin_sort_bins_tag& tag, const int&i ) const { + bool sorted = false; + int upper_bound = bin_offsets(i)+bin_count_const(i); + while(!sorted) { + sorted = true; + int old_idx = sort_order(bin_offsets(i)); + int new_idx; + for(int k=bin_offsets(i)+1; k +struct DefaultBinOp1D { + const int max_bins_; + const double mul_; + typename KeyViewType::const_value_type range_; + typename KeyViewType::const_value_type min_; + + //Construct BinOp with number of bins, minimum value and maxuimum value + DefaultBinOp1D(int max_bins, typename KeyViewType::const_value_type min, + typename KeyViewType::const_value_type max ) + :max_bins_(max_bins+1),mul_(1.0*max_bins/(max-min)),range_(max-min),min_(min) {} + + //Determine bin index from key value + template + KOKKOS_INLINE_FUNCTION + int bin(ViewType& keys, const int& i) const { + return int(mul_*(keys(i)-min_)); + } + + //Return maximum bin index + 1 + KOKKOS_INLINE_FUNCTION + int max_bins() const { + return max_bins_; + } + + //Compare to keys within a bin if true new_val will be put before old_val + template + KOKKOS_INLINE_FUNCTION + bool operator()(ViewType& keys, iType1& i1, iType2& i2) const { + return keys(i1) +struct DefaultBinOp3D { + int max_bins_[3]; + double mul_[3]; + typename KeyViewType::non_const_value_type range_[3]; + typename KeyViewType::non_const_value_type min_[3]; + + DefaultBinOp3D(int max_bins[], typename KeyViewType::const_value_type min[], + typename KeyViewType::const_value_type max[] ) + { + max_bins_[0] = max_bins[0]+1; + max_bins_[1] = max_bins[1]+1; + max_bins_[2] = max_bins[2]+1; + mul_[0] = 1.0*max_bins[0]/(max[0]-min[0]); + mul_[1] = 1.0*max_bins[1]/(max[1]-min[1]); + mul_[2] = 1.0*max_bins[2]/(max[2]-min[2]); + range_[0] = max[0]-min[0]; + range_[1] = max[1]-min[1]; + range_[2] = max[2]-min[2]; + min_[0] = min[0]; + min_[1] = min[1]; + min_[2] = min[2]; + } + + template + KOKKOS_INLINE_FUNCTION + int bin(ViewType& keys, const int& i) const { + return int( (((int(mul_[0]*(keys(i,0)-min_[0]))*max_bins_[1]) + + int(mul_[1]*(keys(i,1)-min_[1])))*max_bins_[2]) + + int(mul_[2]*(keys(i,2)-min_[2]))); + } + + KOKKOS_INLINE_FUNCTION + int max_bins() const { + return max_bins_[0]*max_bins_[1]*max_bins_[2]; + } + + template + KOKKOS_INLINE_FUNCTION + bool operator()(ViewType& keys, iType1& i1 , iType2& i2) const { + if (keys(i1,0)>keys(i2,0)) return true; + else if (keys(i1,0)==keys(i2,0)) { + if (keys(i1,1)>keys(i2,1)) return true; + else if (keys(i1,1)==keys(i2,2)) { + if (keys(i1,2)>keys(i2,2)) return true; + } + } + return false; + } +}; + +template +struct min_max { + Scalar min; + Scalar max; + bool init; + + KOKKOS_INLINE_FUNCTION + min_max() { + min = 0; + max = 0; + init = 0; + } + + KOKKOS_INLINE_FUNCTION + min_max (const min_max& val) { + min = val.min; + max = val.max; + init = val.init; + } + + KOKKOS_INLINE_FUNCTION + min_max operator = (const min_max& val) { + min = val.min; + max = val.max; + init = val.init; + return *this; + } + + KOKKOS_INLINE_FUNCTION + void operator+= (const Scalar& val) { + if(init) { + min = minval?max:val; + } else { + min = val; + max = val; + init = 1; + } + } + + KOKKOS_INLINE_FUNCTION + void operator+= (const min_max& val) { + if(init && val.init) { + min = minval.max?max:val.max; + } else { + if(val.init) { + min = val.min; + max = val.max; + init = 1; + } + } + } + + KOKKOS_INLINE_FUNCTION + void operator+= (volatile const Scalar& val) volatile { + if(init) { + min = minval?max:val; + } else { + min = val; + max = val; + init = 1; + } + } + + KOKKOS_INLINE_FUNCTION + void operator+= (volatile const min_max& val) volatile { + if(init && val.init) { + min = minval.max?max:val.max; + } else { + if(val.init) { + min = val.min; + max = val.max; + init = 1; + } + } + } +}; + + +template +struct min_max_functor { + typedef typename ViewType::execution_space execution_space; + ViewType view; + typedef min_max value_type; + min_max_functor (const ViewType view_):view(view_) { + } + + KOKKOS_INLINE_FUNCTION + void operator()(const size_t& i, value_type& val) const { + val += view(i); + } +}; + +template +bool try_std_sort(ViewType view) { + bool possible = true; + size_t stride[8]; + view.stride(stride); + possible = possible && Impl::is_same::value; + possible = possible && (ViewType::Rank == 1); + possible = possible && (stride[0] == 1); + if(possible) { + std::sort(view.ptr_on_device(),view.ptr_on_device()+view.dimension_0()); + } + return possible; +} + +} + +template +void sort(ViewType view, bool always_use_kokkos_sort = false) { + if(!always_use_kokkos_sort) { + if(SortImpl::try_std_sort(view)) return; + } + + typedef SortImpl::DefaultBinOp1D CompType; + SortImpl::min_max val; + parallel_reduce(view.dimension_0(),SortImpl::min_max_functor(view),val); + BinSort bin_sort(view,CompType(view.dimension_0()/2,val.min,val.max),true); + bin_sort.create_permute_vector(); + bin_sort.sort(view); +} + +/*template +void sort(ViewType view, Comparator comp, bool always_use_kokkos_sort = false) { + +}*/ + +} + +#endif diff --git a/lib/kokkos/containers/src/Kokkos_Bitset.hpp b/lib/kokkos/containers/src/Kokkos_Bitset.hpp new file mode 100755 index 0000000000..b53daab80c --- /dev/null +++ b/lib/kokkos/containers/src/Kokkos_Bitset.hpp @@ -0,0 +1,437 @@ +/* +//@HEADER +// ************************************************************************ +// +// Kokkos: Manycore Performance-Portable Multidimensional Arrays +// Copyright (2012) Sandia Corporation +// +// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +// the U.S. Government retains certain rights in this software. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// 1. Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// 3. Neither the name of the Corporation nor the names of the +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Questions? Contact H. Carter Edwards (hcedwar@sandia.gov) +// +// ************************************************************************ +//@HEADER +*/ + +#ifndef KOKKOS_BITSET_HPP +#define KOKKOS_BITSET_HPP + +#include +#include + +#include + +#include + +namespace Kokkos { + +template +class Bitset; + +template +class ConstBitset; + +template +void deep_copy( Bitset & dst, Bitset const& src); + +template +void deep_copy( Bitset & dst, ConstBitset const& src); + +template +void deep_copy( ConstBitset & dst, ConstBitset const& src); + + +/// A thread safe view to a bitset +template +class Bitset +{ +public: + typedef Device device_type; + typedef unsigned size_type; + + enum { BIT_SCAN_REVERSE = 1u }; + enum { MOVE_HINT_BACKWARD = 2u }; + + enum { + BIT_SCAN_FORWARD_MOVE_HINT_FORWARD = 0u + , BIT_SCAN_REVERSE_MOVE_HINT_FORWARD = BIT_SCAN_REVERSE + , BIT_SCAN_FORWARD_MOVE_HINT_BACKWARD = MOVE_HINT_BACKWARD + , BIT_SCAN_REVERSE_MOVE_HINT_BACKWARD = BIT_SCAN_REVERSE | MOVE_HINT_BACKWARD + }; + +private: + enum { block_size = static_cast(sizeof(unsigned)*CHAR_BIT) }; + enum { block_mask = block_size-1u }; + enum { block_shift = static_cast(Impl::power_of_two::value) }; + +public: + + + /// constructor + /// arg_size := number of bit in set + Bitset(unsigned arg_size = 0u) + : m_size(arg_size) + , m_last_block_mask(0u) + , m_blocks("Bitset", ((m_size + block_mask) >> block_shift) ) + { + for (int i=0, end = static_cast(m_size & block_mask); i < end; ++i) { + m_last_block_mask |= 1u << i; + } + } + + /// assignment + Bitset & operator = (Bitset const & rhs) + { + this->m_size = rhs.m_size; + this->m_last_block_mask = rhs.m_last_block_mask; + this->m_blocks = rhs.m_blocks; + + return *this; + } + + /// copy constructor + Bitset( Bitset const & rhs) + : m_size( rhs.m_size ) + , m_last_block_mask( rhs.m_last_block_mask ) + , m_blocks( rhs.m_blocks ) + {} + + /// number of bits in the set + /// can be call from the host or the device + KOKKOS_FORCEINLINE_FUNCTION + unsigned size() const + { return m_size; } + + /// number of bits which are set to 1 + /// can only be called from the host + unsigned count() const + { + Impl::BitsetCount< Bitset > f(*this); + return f.apply(); + } + + /// set all bits to 1 + /// can only be called from the host + void set() + { + Kokkos::deep_copy(m_blocks, ~0u ); + + if (m_last_block_mask) { + //clear the unused bits in the last block + typedef Kokkos::Impl::DeepCopy< typename device_type::memory_space, Kokkos::HostSpace > raw_deep_copy; + raw_deep_copy( m_blocks.ptr_on_device() + (m_blocks.size() -1u), &m_last_block_mask, sizeof(unsigned)); + } + } + + /// set all bits to 0 + /// can only be called from the host + void reset() + { + Kokkos::deep_copy(m_blocks, 0u ); + } + + /// set all bits to 0 + /// can only be called from the host + void clear() + { + Kokkos::deep_copy(m_blocks, 0u ); + } + + /// set i'th bit to 1 + /// can only be called from the device + KOKKOS_FORCEINLINE_FUNCTION + bool set( unsigned i ) const + { + if ( i < m_size ) { + unsigned * block_ptr = &m_blocks[ i >> block_shift ]; + const unsigned mask = 1u << static_cast( i & block_mask ); + + return !( atomic_fetch_or( block_ptr, mask ) & mask ); + } + return false; + } + + /// set i'th bit to 0 + /// can only be called from the device + KOKKOS_FORCEINLINE_FUNCTION + bool reset( unsigned i ) const + { + if ( i < m_size ) { + unsigned * block_ptr = &m_blocks[ i >> block_shift ]; + const unsigned mask = 1u << static_cast( i & block_mask ); + + return atomic_fetch_and( block_ptr, ~mask ) & mask; + } + return false; + } + + /// return true if the i'th bit set to 1 + /// can only be called from the device + KOKKOS_FORCEINLINE_FUNCTION + bool test( unsigned i ) const + { + if ( i < m_size ) { + const unsigned block = volatile_load(&m_blocks[ i >> block_shift ]); + const unsigned mask = 1u << static_cast( i & block_mask ); + return block & mask; + } + return false; + } + + /// used with find_any_set_near or find_any_unset_near functions + /// returns the max number of times those functions should be call + /// when searching for an available bit + KOKKOS_FORCEINLINE_FUNCTION + unsigned max_hint() const + { + return m_blocks.size(); + } + + /// find a bit set to 1 near the hint + /// returns a pair< bool, unsigned> where if result.first is true then result.second is the bit found + /// and if result.first is false the result.second is a new hint + KOKKOS_INLINE_FUNCTION + Kokkos::pair find_any_set_near( unsigned hint , unsigned scan_direction = BIT_SCAN_FORWARD_MOVE_HINT_FORWARD ) const + { + const unsigned block_idx = (hint >> block_shift) < m_blocks.size() ? (hint >> block_shift) : 0; + const unsigned offset = hint & block_mask; + unsigned block = volatile_load(&m_blocks[ block_idx ]); + block = !m_last_block_mask || (block_idx < (m_blocks.size()-1)) ? block : block & m_last_block_mask ; + + return find_any_helper(block_idx, offset, block, scan_direction); + } + + /// find a bit set to 0 near the hint + /// returns a pair< bool, unsigned> where if result.first is true then result.second is the bit found + /// and if result.first is false the result.second is a new hint + KOKKOS_INLINE_FUNCTION + Kokkos::pair find_any_unset_near( unsigned hint , unsigned scan_direction = BIT_SCAN_FORWARD_MOVE_HINT_FORWARD ) const + { + const unsigned block_idx = hint >> block_shift; + const unsigned offset = hint & block_mask; + unsigned block = volatile_load(&m_blocks[ block_idx ]); + block = !m_last_block_mask || (block_idx < (m_blocks.size()-1) ) ? ~block : ~block & m_last_block_mask ; + + return find_any_helper(block_idx, offset, block, scan_direction); + } + +private: + + KOKKOS_FORCEINLINE_FUNCTION + Kokkos::pair find_any_helper(unsigned block_idx, unsigned offset, unsigned block, unsigned scan_direction) const + { + Kokkos::pair result( block > 0u, 0); + + if (!result.first) { + result.second = update_hint( block_idx, offset, scan_direction ); + } + else { + result.second = scan_block( (block_idx << block_shift) + , offset + , block + , scan_direction + ); + } + return result; + } + + + KOKKOS_FORCEINLINE_FUNCTION + unsigned scan_block(unsigned block_start, int offset, unsigned block, unsigned scan_direction ) const + { + offset = !(scan_direction & BIT_SCAN_REVERSE) ? offset : (offset + block_mask) & block_mask; + block = Impl::rotate_right(block, offset); + return ((( !(scan_direction & BIT_SCAN_REVERSE) ? + Impl::bit_scan_forward(block) : + Impl::bit_scan_reverse(block) + ) + offset + ) & block_mask + ) + block_start; + } + + KOKKOS_FORCEINLINE_FUNCTION + unsigned update_hint( long long block_idx, unsigned offset, unsigned scan_direction ) const + { + block_idx += scan_direction & MOVE_HINT_BACKWARD ? -1 : 1; + block_idx = block_idx >= 0 ? block_idx : m_blocks.size() - 1; + block_idx = block_idx < static_cast(m_blocks.size()) ? block_idx : 0; + + return static_cast(block_idx)*block_size + offset; + } + +private: + + unsigned m_size; + unsigned m_last_block_mask; + View< unsigned *, device_type, MemoryTraits > m_blocks; + +private: + template + friend class Bitset; + + template + friend class ConstBitset; + + template + friend struct Impl::BitsetCount; + + template + friend void deep_copy( Bitset & dst, Bitset const& src); + + template + friend void deep_copy( Bitset & dst, ConstBitset const& src); +}; + +/// a thread-safe view to a const bitset +/// i.e. can only test bits +template +class ConstBitset +{ +public: + typedef Device device_type; + typedef unsigned size_type; + +private: + enum { block_size = static_cast(sizeof(unsigned)*CHAR_BIT) }; + enum { block_mask = block_size -1u }; + enum { block_shift = static_cast(Impl::power_of_two::value) }; + +public: + ConstBitset() + : m_size (0) + {} + + ConstBitset(Bitset const& rhs) + : m_size(rhs.m_size) + , m_blocks(rhs.m_blocks) + {} + + ConstBitset(ConstBitset const& rhs) + : m_size( rhs.m_size ) + , m_blocks( rhs.m_blocks ) + {} + + ConstBitset & operator = (Bitset const & rhs) + { + this->m_size = rhs.m_size; + this->m_blocks = rhs.m_blocks; + + return *this; + } + + ConstBitset & operator = (ConstBitset const & rhs) + { + this->m_size = rhs.m_size; + this->m_blocks = rhs.m_blocks; + + return *this; + } + + + KOKKOS_FORCEINLINE_FUNCTION + unsigned size() const + { + return m_size; + } + + unsigned count() const + { + Impl::BitsetCount< ConstBitset > f(*this); + return f.apply(); + } + + KOKKOS_FORCEINLINE_FUNCTION + bool test( unsigned i ) const + { + if ( i < m_size ) { + const unsigned block = m_blocks[ i >> block_shift ]; + const unsigned mask = 1u << static_cast( i & block_mask ); + return block & mask; + } + return false; + } + +private: + + unsigned m_size; + View< const unsigned *, device_type, MemoryTraits > m_blocks; + +private: + template + friend class ConstBitset; + + template + friend struct Impl::BitsetCount; + + template + friend void deep_copy( Bitset & dst, ConstBitset const& src); + + template + friend void deep_copy( ConstBitset & dst, ConstBitset const& src); +}; + + +template +void deep_copy( Bitset & dst, Bitset const& src) +{ + if (dst.size() != src.size()) { + throw std::runtime_error("Error: Cannot deep_copy bitsets of different sizes!"); + } + + typedef Kokkos::Impl::DeepCopy< typename DstDevice::memory_space, typename SrcDevice::memory_space > raw_deep_copy; + raw_deep_copy(dst.m_blocks.ptr_on_device(), src.m_blocks.ptr_on_device(), sizeof(unsigned)*src.m_blocks.size()); +} + +template +void deep_copy( Bitset & dst, ConstBitset const& src) +{ + if (dst.size() != src.size()) { + throw std::runtime_error("Error: Cannot deep_copy bitsets of different sizes!"); + } + + typedef Kokkos::Impl::DeepCopy< typename DstDevice::memory_space, typename SrcDevice::memory_space > raw_deep_copy; + raw_deep_copy(dst.m_blocks.ptr_on_device(), src.m_blocks.ptr_on_device(), sizeof(unsigned)*src.m_blocks.size()); +} + +template +void deep_copy( ConstBitset & dst, ConstBitset const& src) +{ + if (dst.size() != src.size()) { + throw std::runtime_error("Error: Cannot deep_copy bitsets of different sizes!"); + } + + typedef Kokkos::Impl::DeepCopy< typename DstDevice::memory_space, typename SrcDevice::memory_space > raw_deep_copy; + raw_deep_copy(dst.m_blocks.ptr_on_device(), src.m_blocks.ptr_on_device(), sizeof(unsigned)*src.m_blocks.size()); +} + +} // namespace Kokkos + +#endif //KOKKOS_BITSET_HPP diff --git a/lib/kokkos/containers/src/Kokkos_DualView.hpp b/lib/kokkos/containers/src/Kokkos_DualView.hpp new file mode 100755 index 0000000000..94d3e9ff1b --- /dev/null +++ b/lib/kokkos/containers/src/Kokkos_DualView.hpp @@ -0,0 +1,678 @@ +/* +//@HEADER +// ************************************************************************ +// +// Kokkos +// Manycore Performance-Portable Multidimensional Arrays +// +// Copyright (2012) Sandia Corporation +// +// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +// the U.S. Government retains certain rights in this software. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// 1. Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// 3. Neither the name of the Corporation nor the names of the +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Questions? Contact H. Carter Edwards (hcedwar@sandia.gov) +// +// ************************************************************************ +//@HEADER +*/ + +/// \file Kokkos_DualView.hpp +/// \brief Declaration and definition of Kokkos::DualView. +/// +/// This header file declares and defines Kokkos::DualView and its +/// related nonmember functions. + +#ifndef KOKKOS_DUALVIEW_HPP +#define KOKKOS_DUALVIEW_HPP + +#include +#include + +namespace Kokkos { + +/* \class DualView + * \brief Container to manage mirroring a Kokkos::View that lives + * in device memory with a Kokkos::View that lives in host memory. + * + * This class provides capabilities to manage data which exists in two + * memory spaces at the same time. It keeps views of the same layout + * on two memory spaces as well as modified flags for both + * allocations. Users are responsible for setting the modified flags + * manually if they change the data in either memory space, by calling + * the sync() method templated on the device where they modified the + * data. Users may synchronize data by calling the modify() function, + * templated on the device towards which they want to synchronize + * (i.e., the target of the one-way copy operation). + * + * The DualView class also provides convenience methods such as + * realloc, resize and capacity which call the appropriate methods of + * the underlying Kokkos::View objects. + * + * The four template arguments are the same as those of Kokkos::View. + * (Please refer to that class' documentation for a detailed + * description.) + * + * \tparam DataType The type of the entries stored in the container. + * + * \tparam Layout The array's layout in memory. + * + * \tparam Device The Kokkos Device type. If its memory space is + * not the same as the host's memory space, then DualView will + * contain two separate Views: one in device memory, and one in + * host memory. Otherwise, DualView will only store one View. + * + * \tparam MemoryTraits (optional) The user's intended memory access + * behavior. Please see the documentation of Kokkos::View for + * examples. The default suffices for most users. + */ +template< class DataType , + class Arg1Type = void , + class Arg2Type = void , + class Arg3Type = void> +class DualView : public ViewTraits< DataType , Arg1Type , Arg2Type, Arg3Type > +{ +public: + //! \name Typedefs for device types and various Kokkos::View specializations. + //@{ + typedef ViewTraits< DataType , Arg1Type , Arg2Type, Arg3Type > traits ; + + //! The Kokkos Host Device type; + typedef typename traits::host_mirror_space host_mirror_space ; + + //! The type of a Kokkos::View on the device. + typedef View< typename traits::data_type , + typename traits::array_layout , + typename traits::device_type , + typename traits::memory_traits > t_dev ; + + /// \typedef t_host + /// \brief The type of a Kokkos::View host mirror of \c t_dev. +#if defined( CUDA_VERSION ) && ( 6000 <= CUDA_VERSION ) && defined(KOKKOS_USE_CUDA_UVM) + typedef t_dev t_host; +#else + typedef typename t_dev::HostMirror t_host ; +#endif + + //! The type of a const View on the device. + //! The type of a Kokkos::View on the device. + typedef View< typename traits::const_data_type , + typename traits::array_layout , + typename traits::device_type , + typename traits::memory_traits > t_dev_const ; + + /// \typedef t_host_const + /// \brief The type of a const View host mirror of \c t_dev_const. +#if defined( CUDA_VERSION ) && ( 6000 <= CUDA_VERSION ) && defined(KOKKOS_USE_CUDA_UVM) + typedef t_dev_const t_host_const; +#else + typedef typename t_dev_const::HostMirror t_host_const; +#endif + + //! The type of a const, random-access View on the device. + typedef View< typename traits::const_data_type , + typename traits::array_layout , + typename traits::device_type , + MemoryTraits > t_dev_const_randomread ; + + /// \typedef t_host_const_randomread + /// \brief The type of a const, random-access View host mirror of + /// \c t_dev_const_randomread. +#if defined( CUDA_VERSION ) && ( 6000 <= CUDA_VERSION ) && defined(KOKKOS_USE_CUDA_UVM) + typedef t_dev_const_randomread t_host_const_randomread; +#else + typedef typename t_dev_const_randomread::HostMirror t_host_const_randomread; +#endif + + //! The type of an unmanaged View on the device. + typedef View< typename traits::data_type , + typename traits::array_layout , + typename traits::device_type , + MemoryUnmanaged> t_dev_um; + + //! The type of an unmanaged View host mirror of \c t_dev_um. + typedef View< typename t_host::data_type , + typename t_host::array_layout , + typename t_host::device_type , + MemoryUnmanaged> t_host_um; + + //! The type of a const unmanaged View on the device. + typedef View< typename traits::const_data_type , + typename traits::array_layout , + typename traits::device_type , + MemoryUnmanaged> t_dev_const_um; + + //! The type of a const unmanaged View host mirror of \c t_dev_const_um. + typedef View t_host_const_um; + + //@} + //! \name The two View instances. + //@{ + + t_dev d_view; + t_host h_view; + + //@} + //! \name Counters to keep track of changes ("modified" flags) + //@{ + + View modified_device; + View modified_host; + + //@} + //! \name Constructors + //@{ + + /// \brief Empty constructor. + /// + /// Both device and host View objects are constructed using their + /// default constructors. The "modified" flags are both initialized + /// to "unmodified." + DualView () : + modified_device (View ("DualView::modified_device")), + modified_host (View ("DualView::modified_host")) + {} + + /// \brief Constructor that allocates View objects on both host and device. + /// + /// This constructor works like the analogous constructor of View. + /// The first argument is a string label, which is entirely for your + /// benefit. (Different DualView objects may have the same label if + /// you like.) The arguments that follow are the dimensions of the + /// View objects. For example, if the View has three dimensions, + /// the first three integer arguments will be nonzero, and you may + /// omit the integer arguments that follow. + DualView (const std::string& label, + const size_t n0 = 0, + const size_t n1 = 0, + const size_t n2 = 0, + const size_t n3 = 0, + const size_t n4 = 0, + const size_t n5 = 0, + const size_t n6 = 0, + const size_t n7 = 0) + : d_view (label, n0, n1, n2, n3, n4, n5, n6, n7) +#if defined( CUDA_VERSION ) && ( 6000 <= CUDA_VERSION ) && defined(KOKKOS_USE_CUDA_UVM) + , h_view (d_view) // with UVM, host View is _always_ a shallow copy +#else + , h_view (create_mirror_view (d_view)) // without UVM, host View mirrors +#endif + , modified_device (View ("DualView::modified_device")) + , modified_host (View ("DualView::modified_host")) + {} + + //! Copy constructor (shallow copy) + template + DualView (const DualView& src) : + d_view (src.d_view), + h_view (src.h_view), + modified_device (src.modified_device), + modified_host (src.modified_host) + {} + + /// \brief Create DualView from existing device and host View objects. + /// + /// This constructor assumes that the device and host View objects + /// are synchronized. You, the caller, are responsible for making + /// sure this is the case before calling this constructor. After + /// this constructor returns, you may use DualView's sync() and + /// modify() methods to ensure synchronization of the View objects. + /// + /// \param d_view_ Device View + /// \param h_view_ Host View (must have type t_host = t_dev::HostMirror) + DualView (const t_dev& d_view_, const t_host& h_view_) : + d_view (d_view_), + h_view (h_view_), + modified_device (View ("DualView::modified_device")), + modified_host (View ("DualView::modified_host")) + { + Impl::assert_shapes_are_equal (d_view.shape (), h_view.shape ()); + } + + //@} + //! \name Methods for synchronizing, marking as modified, and getting Views. + //@{ + + /// \brief Return a View on a specific device \c Device. + /// + /// Please don't be afraid of the if_c expression in the return + /// value's type. That just tells the method what the return type + /// should be: t_dev if the \c Device template parameter matches + /// this DualView's device type, else t_host. + /// + /// For example, suppose you create a DualView on Cuda, like this: + /// \code + /// typedef Kokkos::DualView dual_view_type; + /// dual_view_type DV ("my dual view", 100); + /// \endcode + /// If you want to get the CUDA device View, do this: + /// \code + /// typename dual_view_type::t_dev cudaView = DV.view (); + /// \endcode + /// and if you want to get the host mirror of that View, do this: + /// \code + /// typedef typename Kokkos::HostSpace::execution_space host_device_type; + /// typename dual_view_type::t_host hostView = DV.view (); + /// \endcode + template< class Device > + KOKKOS_INLINE_FUNCTION + const typename Impl::if_c< + Impl::is_same::value, + t_dev, + t_host>::type& view () const + { + return Impl::if_c< + Impl::is_same< + typename t_dev::memory_space, + typename Device::memory_space>::value, + t_dev, + t_host >::select (d_view , h_view); + } + + /// \brief Update data on device or host only if data in the other + /// space has been marked as modified. + /// + /// If \c Device is the same as this DualView's device type, then + /// copy data from host to device. Otherwise, copy data from device + /// to host. In either case, only copy if the source of the copy + /// has been modified. + /// + /// This is a one-way synchronization only. If the target of the + /// copy has been modified, this operation will discard those + /// modifications. It will also reset both device and host modified + /// flags. + /// + /// \note This method doesn't know on its own whether you modified + /// the data in either View. You must manually mark modified data + /// as modified, by calling the modify() method with the + /// appropriate template parameter. + template + void sync( const typename Impl::enable_if< + ( Impl::is_same< typename traits::data_type , typename traits::non_const_data_type>::value) || + ( Impl::is_same< Device , int>::value) + , int >::type& = 0) + { + const unsigned int dev = + Impl::if_c< + Impl::is_same< + typename t_dev::memory_space, + typename Device::memory_space>::value , + unsigned int, + unsigned int>::select (1, 0); + + if (dev) { // if Device is the same as DualView's device type + if ((modified_host () > 0) && (modified_host () >= modified_device ())) { + deep_copy (d_view, h_view); + modified_host() = modified_device() = 0; + } + } else { // hopefully Device is the same as DualView's host type + if ((modified_device () > 0) && (modified_device () >= modified_host ())) { + deep_copy (h_view, d_view); + modified_host() = modified_device() = 0; + } + } + } + + template + void sync ( const typename Impl::enable_if< + ( ! Impl::is_same< typename traits::data_type , typename traits::non_const_data_type>::value ) || + ( Impl::is_same< Device , int>::value) + , int >::type& = 0 ) + { + const unsigned int dev = + Impl::if_c< + Impl::is_same< + typename t_dev::memory_space, + typename Device::memory_space>::value, + unsigned int, + unsigned int>::select (1, 0); + if (dev) { // if Device is the same as DualView's device type + if ((modified_host () > 0) && (modified_host () >= modified_device ())) { + Impl::throw_runtime_exception("Calling sync on a DualView with a const datatype."); + } + } else { // hopefully Device is the same as DualView's host type + if ((modified_device () > 0) && (modified_device () >= modified_host ())) { + Impl::throw_runtime_exception("Calling sync on a DualView with a const datatype."); + } + } + } + /// \brief Mark data as modified on the given device \c Device. + /// + /// If \c Device is the same as this DualView's device type, then + /// mark the device's data as modified. Otherwise, mark the host's + /// data as modified. + template + void modify () { + const unsigned int dev = + Impl::if_c< + Impl::is_same< + typename t_dev::memory_space, + typename Device::memory_space>::value, + unsigned int, + unsigned int>::select (1, 0); + + if (dev) { // if Device is the same as DualView's device type + // Increment the device's modified count. + modified_device () = (modified_device () > modified_host () ? + modified_device () : modified_host ()) + 1; + } else { // hopefully Device is the same as DualView's host type + // Increment the host's modified count. + modified_host () = (modified_device () > modified_host () ? + modified_device () : modified_host ()) + 1; + } + } + + //@} + //! \name Methods for reallocating or resizing the View objects. + //@{ + + /// \brief Reallocate both View objects. + /// + /// This discards any existing contents of the objects, and resets + /// their modified flags. It does not copy the old contents + /// of either View into the new View objects. + void realloc( const size_t n0 = 0 , + const size_t n1 = 0 , + const size_t n2 = 0 , + const size_t n3 = 0 , + const size_t n4 = 0 , + const size_t n5 = 0 , + const size_t n6 = 0 , + const size_t n7 = 0 ) { + ::Kokkos::realloc(d_view,n0,n1,n2,n3,n4,n5,n6,n7); +#if defined( CUDA_VERSION ) && ( 6000 <= CUDA_VERSION ) && defined(KOKKOS_USE_CUDA_UVM) + h_view = d_view ; +#else + h_view = create_mirror_view( d_view ); +#endif + /* Reset dirty flags */ + modified_device() = modified_host() = 0; + } + + /// \brief Resize both views, copying old contents into new if necessary. + /// + /// This method only copies the old contents into the new View + /// objects for the device which was last marked as modified. + void resize( const size_t n0 = 0 , + const size_t n1 = 0 , + const size_t n2 = 0 , + const size_t n3 = 0 , + const size_t n4 = 0 , + const size_t n5 = 0 , + const size_t n6 = 0 , + const size_t n7 = 0 ) { + if(modified_device() >= modified_host()) { + /* Resize on Device */ + ::Kokkos::resize(d_view,n0,n1,n2,n3,n4,n5,n6,n7); +#if defined( CUDA_VERSION ) && ( 6000 <= CUDA_VERSION ) && defined(KOKKOS_USE_CUDA_UVM) + h_view = d_view ; +#else + h_view = create_mirror_view( d_view ); +#endif + + /* Mark Device copy as modified */ + modified_device() = modified_device()+1; + + } else { + /* Realloc on Device */ + + ::Kokkos::realloc(d_view,n0,n1,n2,n3,n4,n5,n6,n7); +#if defined( CUDA_VERSION ) && ( 6000 <= CUDA_VERSION ) && defined(KOKKOS_USE_CUDA_UVM) + t_host temp_view = d_view ; +#else + t_host temp_view = create_mirror_view( d_view ); +#endif + + /* Remap on Host */ + Impl::ViewRemap< t_host , t_host >( temp_view , h_view ); + h_view = temp_view; + + /* Mark Host copy as modified */ + modified_host() = modified_host()+1; + } + } + + //@} + //! \name Methods for getting capacity, stride, or dimension(s). + //@{ + + //! The allocation size (same as Kokkos::View::capacity). + size_t capacity() const { + return d_view.capacity(); + } + + //! Get stride(s) for each dimension. + template< typename iType> + void stride(iType* stride_) const { + d_view.stride(stride_); + } + + /* \brief return size of dimension 0 */ + size_t dimension_0() const {return d_view.dimension_0();} + /* \brief return size of dimension 1 */ + size_t dimension_1() const {return d_view.dimension_1();} + /* \brief return size of dimension 2 */ + size_t dimension_2() const {return d_view.dimension_2();} + /* \brief return size of dimension 3 */ + size_t dimension_3() const {return d_view.dimension_3();} + /* \brief return size of dimension 4 */ + size_t dimension_4() const {return d_view.dimension_4();} + /* \brief return size of dimension 5 */ + size_t dimension_5() const {return d_view.dimension_5();} + /* \brief return size of dimension 6 */ + size_t dimension_6() const {return d_view.dimension_6();} + /* \brief return size of dimension 7 */ + size_t dimension_7() const {return d_view.dimension_7();} + + //@} +}; + +// +// Partial specializations of Kokkos::subview() for DualView objects. +// + +template< class DstViewType , + class T , class L , class D , class M , + class ArgType0 > +DstViewType +subview( const DualView & src , + const ArgType0 & arg0 ) +{ + DstViewType sub_view; + sub_view.d_view = subview(src.d_view,arg0); + sub_view.h_view = subview(src.h_view,arg0); + sub_view.modified_device = src.modified_device; + sub_view.modified_host = src.modified_host; + return sub_view; +} + + +template< class DstViewType , + class T , class L , class D , class M , + class ArgType0 , class ArgType1 > +DstViewType +subview( const DualView & src , + const ArgType0 & arg0 , + const ArgType1 & arg1 ) +{ + DstViewType sub_view; + sub_view.d_view = subview(src.d_view,arg0,arg1); + sub_view.h_view = subview(src.h_view,arg0,arg1); + sub_view.modified_device = src.modified_device; + sub_view.modified_host = src.modified_host; + return sub_view; +} + +template< class DstViewType , + class T , class L , class D , class M , + class ArgType0 , class ArgType1 , class ArgType2 > +DstViewType +subview( const DualView & src , + const ArgType0 & arg0 , + const ArgType1 & arg1 , + const ArgType2 & arg2 ) +{ + DstViewType sub_view; + sub_view.d_view = subview(src.d_view,arg0,arg1,arg2); + sub_view.h_view = subview(src.h_view,arg0,arg1,arg2); + sub_view.modified_device = src.modified_device; + sub_view.modified_host = src.modified_host; + return sub_view; +} + +template< class DstViewType , + class T , class L , class D , class M , + class ArgType0 , class ArgType1 , class ArgType2 , class ArgType3 > +DstViewType +subview( const DualView & src , + const ArgType0 & arg0 , + const ArgType1 & arg1 , + const ArgType2 & arg2 , + const ArgType3 & arg3 ) +{ + DstViewType sub_view; + sub_view.d_view = subview(src.d_view,arg0,arg1,arg2,arg3); + sub_view.h_view = subview(src.h_view,arg0,arg1,arg2,arg3); + sub_view.modified_device = src.modified_device; + sub_view.modified_host = src.modified_host; + return sub_view; +} + +template< class DstViewType , + class T , class L , class D , class M , + class ArgType0 , class ArgType1 , class ArgType2 , class ArgType3 , + class ArgType4 > +DstViewType +subview( const DualView & src , + const ArgType0 & arg0 , + const ArgType1 & arg1 , + const ArgType2 & arg2 , + const ArgType3 & arg3 , + const ArgType4 & arg4 ) +{ + DstViewType sub_view; + sub_view.d_view = subview(src.d_view,arg0,arg1,arg2,arg3,arg4); + sub_view.h_view = subview(src.h_view,arg0,arg1,arg2,arg3,arg4); + sub_view.modified_device = src.modified_device; + sub_view.modified_host = src.modified_host; + return sub_view; +} + +template< class DstViewType , + class T , class L , class D , class M , + class ArgType0 , class ArgType1 , class ArgType2 , class ArgType3 , + class ArgType4 , class ArgType5 > +DstViewType +subview( const DualView & src , + const ArgType0 & arg0 , + const ArgType1 & arg1 , + const ArgType2 & arg2 , + const ArgType3 & arg3 , + const ArgType4 & arg4 , + const ArgType5 & arg5 ) +{ + DstViewType sub_view; + sub_view.d_view = subview(src.d_view,arg0,arg1,arg2,arg3,arg4,arg5); + sub_view.h_view = subview(src.h_view,arg0,arg1,arg2,arg3,arg4,arg5); + sub_view.modified_device = src.modified_device; + sub_view.modified_host = src.modified_host; + return sub_view; +} + +template< class DstViewType , + class T , class L , class D , class M , + class ArgType0 , class ArgType1 , class ArgType2 , class ArgType3 , + class ArgType4 , class ArgType5 , class ArgType6 > +DstViewType +subview( const DualView & src , + const ArgType0 & arg0 , + const ArgType1 & arg1 , + const ArgType2 & arg2 , + const ArgType3 & arg3 , + const ArgType4 & arg4 , + const ArgType5 & arg5 , + const ArgType6 & arg6 ) +{ + DstViewType sub_view; + sub_view.d_view = subview(src.d_view,arg0,arg1,arg2,arg3,arg4,arg5,arg6); + sub_view.h_view = subview(src.h_view,arg0,arg1,arg2,arg3,arg4,arg5,arg6); + sub_view.modified_device = src.modified_device; + sub_view.modified_host = src.modified_host; + return sub_view; +} + +template< class DstViewType , + class T , class L , class D , class M , + class ArgType0 , class ArgType1 , class ArgType2 , class ArgType3 , + class ArgType4 , class ArgType5 , class ArgType6 , class ArgType7 > +DstViewType +subview( const DualView & src , + const ArgType0 & arg0 , + const ArgType1 & arg1 , + const ArgType2 & arg2 , + const ArgType3 & arg3 , + const ArgType4 & arg4 , + const ArgType5 & arg5 , + const ArgType6 & arg6 , + const ArgType7 & arg7 ) +{ + DstViewType sub_view; + sub_view.d_view = subview(src.d_view,arg0,arg1,arg2,arg3,arg4,arg5,arg6,arg7); + sub_view.h_view = subview(src.h_view,arg0,arg1,arg2,arg3,arg4,arg5,arg6,arg7); + sub_view.modified_device = src.modified_device; + sub_view.modified_host = src.modified_host; + return sub_view; +} + +// +// Partial specialization of Kokkos::deep_copy() for DualView objects. +// + +template< class DT , class DL , class DD , class DM , + class ST , class SL , class SD , class SM > +void +deep_copy (DualView dst, // trust me, this must not be a reference + const DualView& src ) +{ + if (src.modified_device () >= src.modified_host ()) { + deep_copy (dst.d_view, src.d_view); + dst.template modify::device_type> (); + } else { + deep_copy (dst.h_view, src.h_view); + dst.template modify::host_mirror_space> (); + } +} + +} // namespace Kokkos + +#endif diff --git a/lib/kokkos/containers/src/Kokkos_Functional.hpp b/lib/kokkos/containers/src/Kokkos_Functional.hpp new file mode 100755 index 0000000000..74c3f7093c --- /dev/null +++ b/lib/kokkos/containers/src/Kokkos_Functional.hpp @@ -0,0 +1,132 @@ +#ifndef KOKKOS_FUNCTIONAL_HPP +#define KOKKOS_FUNCTIONAL_HPP + +#include +#include + +namespace Kokkos { + +// These should work for most types + +template +struct pod_hash +{ + typedef T argument_type; + typedef T first_argument_type; + typedef uint32_t second_argument_type; + typedef uint32_t result_type; + + KOKKOS_FORCEINLINE_FUNCTION + uint32_t operator()(T const & t) const + { return Impl::MurmurHash3_x86_32( &t, sizeof(T), 0); } + + KOKKOS_FORCEINLINE_FUNCTION + uint32_t operator()(T const & t, uint32_t seed) const + { return Impl::MurmurHash3_x86_32( &t, sizeof(T), seed); } +}; + +template +struct pod_equal_to +{ + typedef T first_argument_type; + typedef T second_argument_type; + typedef bool result_type; + + KOKKOS_FORCEINLINE_FUNCTION + bool operator()(T const & a, T const & b) const + { return Impl::bitwise_equal(&a,&b); } +}; + +template +struct pod_not_equal_to +{ + typedef T first_argument_type; + typedef T second_argument_type; + typedef bool result_type; + + KOKKOS_FORCEINLINE_FUNCTION + bool operator()(T const & a, T const & b) const + { return !Impl::bitwise_equal(&a,&b); } +}; + +template +struct equal_to +{ + typedef T first_argument_type; + typedef T second_argument_type; + typedef bool result_type; + + KOKKOS_FORCEINLINE_FUNCTION + bool operator()(T const & a, T const & b) const + { return a == b; } +}; + +template +struct not_equal_to +{ + typedef T first_argument_type; + typedef T second_argument_type; + typedef bool result_type; + + KOKKOS_FORCEINLINE_FUNCTION + bool operator()(T const & a, T const & b) const + { return a != b; } +}; + + +template +struct greater +{ + typedef T first_argument_type; + typedef T second_argument_type; + typedef bool result_type; + + KOKKOS_FORCEINLINE_FUNCTION + bool operator()(T const & a, T const & b) const + { return a > b; } +}; + + +template +struct less +{ + typedef T first_argument_type; + typedef T second_argument_type; + typedef bool result_type; + + KOKKOS_FORCEINLINE_FUNCTION + bool operator()(T const & a, T const & b) const + { return a < b; } +}; + +template +struct greater_equal +{ + typedef T first_argument_type; + typedef T second_argument_type; + typedef bool result_type; + + KOKKOS_FORCEINLINE_FUNCTION + bool operator()(T const & a, T const & b) const + { return a >= b; } +}; + + +template +struct less_equal +{ + typedef T first_argument_type; + typedef T second_argument_type; + typedef bool result_type; + + KOKKOS_FORCEINLINE_FUNCTION + bool operator()(T const & a, T const & b) const + { return a <= b; } +}; + +} // namespace Kokkos + + +#endif //KOKKOS_FUNCTIONAL_HPP + + diff --git a/lib/kokkos/containers/src/Kokkos_SegmentedView.hpp b/lib/kokkos/containers/src/Kokkos_SegmentedView.hpp new file mode 100755 index 0000000000..6730757b31 --- /dev/null +++ b/lib/kokkos/containers/src/Kokkos_SegmentedView.hpp @@ -0,0 +1,478 @@ +/* +//@HEADER +// ************************************************************************ +// +// Kokkos: Manycore Performance-Portable Multidimensional Arrays +// Copyright (2012) Sandia Corporation +// +// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +// the U.S. Government retains certain rights in this software. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// 1. Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// 3. Neither the name of the Corporation nor the names of the +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Questions? Contact H. Carter Edwards (hcedwar@sandia.gov) +// +// ************************************************************************ +//@HEADER +*/ + +#ifndef KOKKOS_SEGMENTED_VIEW_HPP_ +#define KOKKOS_SEGMENTED_VIEW_HPP_ + +#include +#include +#include + +namespace Kokkos { + +namespace Impl { + +template +struct delete_segmented_view; + +template +inline +void DeviceSetAllocatableMemorySize(size_t) {} + +#if defined( KOKKOS_HAVE_CUDA ) + +template<> +inline +void DeviceSetAllocatableMemorySize(size_t size) { +#ifdef __CUDACC__ + size_t size_limit; + cudaDeviceGetLimit(&size_limit,cudaLimitMallocHeapSize); + if(size_limit +inline +void DeviceSetAllocatableMemorySize(size_t size) { +#ifdef __CUDACC__ + size_t size_limit; + cudaDeviceGetLimit(&size_limit,cudaLimitMallocHeapSize); + if(size_limit +class SegmentedView : public ViewTraits< DataType , Arg1Type , Arg2Type, Arg3Type > +{ +public: + //! \name Typedefs for device types and various Kokkos::View specializations. + //@{ + typedef ViewTraits< DataType , Arg1Type , Arg2Type, Arg3Type > traits ; + + //! The type of a Kokkos::View on the device. + typedef View< typename traits::data_type , + typename traits::array_layout , + typename traits::memory_space , + Kokkos::MemoryUnmanaged > t_dev ; + + +private: + Kokkos::View segments_; + + Kokkos::View realloc_lock; + Kokkos::View nsegments_; + + size_t segment_length_; + size_t segment_length_m1_; + int max_segments_; + + int segment_length_log2; + + // Dimensions, cardinality, capacity, and offset computation for + // multidimensional array view of contiguous memory. + // Inherits from Impl::Shape + typedef Impl::ViewOffset< typename traits::shape_type + , typename traits::array_layout + > offset_map_type ; + + offset_map_type m_offset_map ; + + typedef View< typename traits::array_intrinsic_type , + typename traits::array_layout , + typename traits::memory_space , + typename traits::memory_traits > array_type ; + + typedef View< typename traits::const_data_type , + typename traits::array_layout , + typename traits::memory_space , + typename traits::memory_traits > const_type ; + + typedef View< typename traits::non_const_data_type , + typename traits::array_layout , + typename traits::memory_space , + typename traits::memory_traits > non_const_type ; + + typedef View< typename traits::non_const_data_type , + typename traits::array_layout , + HostSpace , + void > HostMirror ; + + template< bool Accessible > + KOKKOS_INLINE_FUNCTION + typename Impl::enable_if< Accessible , typename traits::size_type >::type + dimension_0_intern() const { return nsegments_() * segment_length_ ; } + + template< bool Accessible > + KOKKOS_INLINE_FUNCTION + typename Impl::enable_if< ! Accessible , typename traits::size_type >::type + dimension_0_intern() const + { + // In Host space + int n = 0 ; +#if ! defined( __CUDA_ARCH__ ) + Impl::DeepCopy< HostSpace , typename traits::memory_space >( & n , nsegments_.ptr_on_device() , sizeof(int) ); +#endif + + return n * segment_length_ ; + } + +public: + + enum { Rank = traits::rank }; + + KOKKOS_INLINE_FUNCTION offset_map_type shape() const { return m_offset_map ; } + + /* \brief return (current) size of dimension 0 */ + KOKKOS_INLINE_FUNCTION typename traits::size_type dimension_0() const { + enum { Accessible = Impl::VerifyExecutionCanAccessMemorySpace< + Impl::ActiveExecutionMemorySpace, typename traits::memory_space >::value }; + int n = SegmentedView::dimension_0_intern< Accessible >(); + return n ; + } + + /* \brief return size of dimension 1 */ + KOKKOS_INLINE_FUNCTION typename traits::size_type dimension_1() const { return m_offset_map.N1 ; } + /* \brief return size of dimension 2 */ + KOKKOS_INLINE_FUNCTION typename traits::size_type dimension_2() const { return m_offset_map.N2 ; } + /* \brief return size of dimension 3 */ + KOKKOS_INLINE_FUNCTION typename traits::size_type dimension_3() const { return m_offset_map.N3 ; } + /* \brief return size of dimension 4 */ + KOKKOS_INLINE_FUNCTION typename traits::size_type dimension_4() const { return m_offset_map.N4 ; } + /* \brief return size of dimension 5 */ + KOKKOS_INLINE_FUNCTION typename traits::size_type dimension_5() const { return m_offset_map.N5 ; } + /* \brief return size of dimension 6 */ + KOKKOS_INLINE_FUNCTION typename traits::size_type dimension_6() const { return m_offset_map.N6 ; } + /* \brief return size of dimension 7 */ + KOKKOS_INLINE_FUNCTION typename traits::size_type dimension_7() const { return m_offset_map.N7 ; } + + /* \brief return size of dimension 2 */ + KOKKOS_INLINE_FUNCTION typename traits::size_type size() const { + return dimension_0() * + m_offset_map.N1 * m_offset_map.N2 * m_offset_map.N3 * m_offset_map.N4 * + m_offset_map.N5 * m_offset_map.N6 * m_offset_map.N7 ; + } + + template< typename iType > + KOKKOS_INLINE_FUNCTION + typename traits::size_type dimension( const iType & i ) const { + if(i==0) + return dimension_0(); + else + return Impl::dimension( m_offset_map , i ); + } + + KOKKOS_INLINE_FUNCTION + typename traits::size_type capacity() { + return segments_.dimension_0() * + m_offset_map.N1 * m_offset_map.N2 * m_offset_map.N3 * m_offset_map.N4 * + m_offset_map.N5 * m_offset_map.N6 * m_offset_map.N7; + } + + KOKKOS_INLINE_FUNCTION + typename traits::size_type get_num_segments() { + enum { Accessible = Impl::VerifyExecutionCanAccessMemorySpace< + Impl::ActiveExecutionMemorySpace, typename traits::memory_space >::value }; + int n = SegmentedView::dimension_0_intern< Accessible >(); + return n/segment_length_ ; + } + + KOKKOS_INLINE_FUNCTION + typename traits::size_type get_max_segments() { + return max_segments_; + } + + /// \brief Constructor that allocates View objects with an initial length of 0. + /// + /// This constructor works mostly like the analogous constructor of View. + /// The first argument is a string label, which is entirely for your + /// benefit. (Different SegmentedView objects may have the same label if + /// you like.) The second argument 'view_length' is the size of the segments. + /// This number must be a power of two. The third argument n0 is the maximum + /// value for the first dimension of the segmented view. The maximal allocatable + /// number of Segments is thus: (n0+view_length-1)/view_length. + /// The arguments that follow are the other dimensions of the (1-7) of the + /// View objects. For example, for a View with 3 runtime dimensions, + /// the first 4 integer arguments will be nonzero: + /// SegmentedView("Name",32768,10000000,8,4). This allocates a SegmentedView + /// with a maximum of 306 segments of dimension (32768,8,4). The logical size of + /// the segmented view is (n,8,4) with n between 0 and 10000000. + /// You may omit the integer arguments that follow. + template< class LabelType > + SegmentedView(const LabelType & label , + const size_t view_length , + const size_t n0 , + const size_t n1 = 0 , + const size_t n2 = 0 , + const size_t n3 = 0 , + const size_t n4 = 0 , + const size_t n5 = 0 , + const size_t n6 = 0 , + const size_t n7 = 0 + ): segment_length_(view_length),segment_length_m1_(view_length-1) + { + segment_length_log2 = -1; + size_t l = segment_length_; + while(l>0) { + l>>=1; + segment_length_log2++; + } + l = 1<(segment_length_*max_segments_*sizeof(typename traits::value_type)); + + segments_ = Kokkos::View(label , max_segments_); + realloc_lock = Kokkos::View("Lock"); + nsegments_ = Kokkos::View("nviews"); + m_offset_map.assign( n0, n1, n2, n3, n4, n5, n6, n7, n0*n1*n2*n3*n4*n5*n6*n7 ); + + } + + KOKKOS_INLINE_FUNCTION + SegmentedView(const SegmentedView& src): + segments_(src.segments_), + realloc_lock (src.realloc_lock), + nsegments_ (src.nsegments_), + segment_length_(src.segment_length_), + segment_length_m1_(src.segment_length_m1_), + max_segments_ (src.max_segments_), + segment_length_log2(src.segment_length_log2), + m_offset_map (src.m_offset_map) + {} + + KOKKOS_INLINE_FUNCTION + SegmentedView& operator= (const SegmentedView& src) { + segments_ = src.segments_; + realloc_lock = src.realloc_lock; + nsegments_ = src.nsegments_; + segment_length_= src.segment_length_; + segment_length_m1_= src.segment_length_m1_; + max_segments_ = src.max_segments_; + segment_length_log2= src.segment_length_log2; + m_offset_map = src.m_offset_map; + return *this; + } + + ~SegmentedView() { + if (traits::execution_space::in_parallel()) return; + int count = traits::memory_space::count(segments_.ptr_on_device()); + if(count == 1) { + Kokkos::fence(); + typename Kokkos::View::HostMirror h_nviews("h_nviews"); + Kokkos::deep_copy(h_nviews,nsegments_); + Kokkos::parallel_for(h_nviews(),Impl::delete_segmented_view(*this)); + } + } + + KOKKOS_INLINE_FUNCTION + t_dev get_segment(const int& i) const { + return segments_[i]; + } + + template< class MemberType> + KOKKOS_INLINE_FUNCTION + void grow (MemberType& team_member, const size_t& growSize) const { + if (growSize>max_segments_*segment_length_) { + printf ("Exceeding maxSize: %lu %lu\n", growSize, max_segments_*segment_length_); + return; + } + if(team_member.team_rank()==0) { + bool too_small = growSize > segment_length_ * nsegments_(); + while(too_small && Kokkos::atomic_compare_exchange(&realloc_lock(),0,1) ) { + too_small = growSize > segment_length_ * nsegments_(); + } + if(too_small) { + while(too_small) { + const size_t alloc_size = segment_length_*m_offset_map.N1*m_offset_map.N2*m_offset_map.N3* + m_offset_map.N4*m_offset_map.N5*m_offset_map.N6*m_offset_map.N7; + typename traits::non_const_value_type* const ptr = new typename traits::non_const_value_type[alloc_size]; + + segments_(nsegments_()) = + t_dev(ptr,segment_length_,m_offset_map.N1,m_offset_map.N2,m_offset_map.N3,m_offset_map.N4,m_offset_map.N5,m_offset_map.N6,m_offset_map.N7); + nsegments_()++; + too_small = growSize > segment_length_ * nsegments_(); + } + realloc_lock() = 0; + } + } + team_member.team_barrier(); + } + + KOKKOS_INLINE_FUNCTION + void grow_non_thread_safe (const size_t& growSize) const { + if (growSize>max_segments_*segment_length_) { + printf ("Exceeding maxSize: %lu %lu\n", growSize, max_segments_*segment_length_); + return; + } + bool too_small = growSize > segment_length_ * nsegments_(); + if(too_small) { + while(too_small) { + const size_t alloc_size = segment_length_*m_offset_map.N1*m_offset_map.N2*m_offset_map.N3* + m_offset_map.N4*m_offset_map.N5*m_offset_map.N6*m_offset_map.N7; + typename traits::non_const_value_type* const ptr = + new typename traits::non_const_value_type[alloc_size]; + + segments_(nsegments_()) = + t_dev (ptr, segment_length_, m_offset_map.N1, m_offset_map.N2, + m_offset_map.N3, m_offset_map.N4, m_offset_map.N5, + m_offset_map.N6, m_offset_map.N7); + nsegments_()++; + too_small = growSize > segment_length_ * nsegments_(); + } + } + } + + template< typename iType0 > + KOKKOS_FORCEINLINE_FUNCTION + typename Impl::ViewEnableArrayOper< typename traits::value_type & , traits, typename traits::array_layout, 1, iType0 >::type + operator() ( const iType0 & i0 ) const + { + return segments_[i0>>segment_length_log2](i0&(segment_length_m1_)); + } + + template< typename iType0 , typename iType1 > + KOKKOS_FORCEINLINE_FUNCTION + typename Impl::ViewEnableArrayOper< typename traits::value_type & , traits, typename traits::array_layout, 2, + iType0 , iType1>::type + operator() ( const iType0 & i0 , const iType1 & i1 ) const + { + return segments_[i0>>segment_length_log2](i0&(segment_length_m1_),i1); + } + + template< typename iType0 , typename iType1 , typename iType2 > + KOKKOS_FORCEINLINE_FUNCTION + typename Impl::ViewEnableArrayOper< typename traits::value_type & , traits, typename traits::array_layout, 3, + iType0 , iType1 , iType2 >::type + operator() ( const iType0 & i0 , const iType1 & i1 , const iType2 & i2 ) const + { + return segments_[i0>>segment_length_log2](i0&(segment_length_m1_),i1,i2); + } + + template< typename iType0 , typename iType1 , typename iType2 , typename iType3 > + KOKKOS_FORCEINLINE_FUNCTION + typename Impl::ViewEnableArrayOper< typename traits::value_type & , traits, typename traits::array_layout, 4, + iType0 , iType1 , iType2 , iType3 >::type + operator() ( const iType0 & i0 , const iType1 & i1 , const iType2 & i2 , const iType3 & i3 ) const + { + return segments_[i0>>segment_length_log2](i0&(segment_length_m1_),i1,i2,i3); + } + + template< typename iType0 , typename iType1 , typename iType2 , typename iType3 , + typename iType4 > + KOKKOS_FORCEINLINE_FUNCTION + typename Impl::ViewEnableArrayOper< typename traits::value_type & , traits, typename traits::array_layout, 5, + iType0 , iType1 , iType2 , iType3 , iType4 >::type + operator() ( const iType0 & i0 , const iType1 & i1 , const iType2 & i2 , const iType3 & i3 , + const iType4 & i4 ) const + { + return segments_[i0>>segment_length_log2](i0&(segment_length_m1_),i1,i2,i3,i4); + } + + template< typename iType0 , typename iType1 , typename iType2 , typename iType3 , + typename iType4 , typename iType5 > + KOKKOS_FORCEINLINE_FUNCTION + typename Impl::ViewEnableArrayOper< typename traits::value_type & , traits, typename traits::array_layout, 6, + iType0 , iType1 , iType2 , iType3 , iType4 , iType5>::type + operator() ( const iType0 & i0 , const iType1 & i1 , const iType2 & i2 , const iType3 & i3 , + const iType4 & i4 , const iType5 & i5 ) const + { + return segments_[i0>>segment_length_log2](i0&(segment_length_m1_),i1,i2,i3,i4,i5); + } + + template< typename iType0 , typename iType1 , typename iType2 , typename iType3 , + typename iType4 , typename iType5 , typename iType6 > + KOKKOS_FORCEINLINE_FUNCTION + typename Impl::ViewEnableArrayOper< typename traits::value_type & , traits, typename traits::array_layout, 7, + iType0 , iType1 , iType2 , iType3 , iType4 , iType5 , iType6>::type + operator() ( const iType0 & i0 , const iType1 & i1 , const iType2 & i2 , const iType3 & i3 , + const iType4 & i4 , const iType5 & i5 , const iType6 & i6 ) const + { + return segments_[i0>>segment_length_log2](i0&(segment_length_m1_),i1,i2,i3,i4,i5,i6); + } + + template< typename iType0 , typename iType1 , typename iType2 , typename iType3 , + typename iType4 , typename iType5 , typename iType6 , typename iType7 > + KOKKOS_FORCEINLINE_FUNCTION + typename Impl::ViewEnableArrayOper< typename traits::value_type & , traits, typename traits::array_layout, 8, + iType0 , iType1 , iType2 , iType3 , iType4 , iType5 , iType6 , iType7>::type + operator() ( const iType0 & i0 , const iType1 & i1 , const iType2 & i2 , const iType3 & i3 , + const iType4 & i4 , const iType5 & i5 , const iType6 & i6 , const iType7 & i7 ) const + { + return segments_[i0>>segment_length_log2](i0&(segment_length_m1_),i1,i2,i3,i4,i5,i6,i7); + } +}; + +namespace Impl { +template +struct delete_segmented_view { + typedef SegmentedView view_type; + typedef typename view_type::execution_space device_type; + + view_type view_; + delete_segmented_view(view_type view):view_(view) { + } + + KOKKOS_INLINE_FUNCTION + void operator() (int i) const { + delete [] view_.get_segment(i).ptr_on_device(); + } +}; + +} +} + +#endif diff --git a/lib/kokkos/containers/src/Kokkos_StaticCrsGraph.hpp b/lib/kokkos/containers/src/Kokkos_StaticCrsGraph.hpp new file mode 100755 index 0000000000..44bd1da32d --- /dev/null +++ b/lib/kokkos/containers/src/Kokkos_StaticCrsGraph.hpp @@ -0,0 +1,225 @@ +/* +//@HEADER +// ************************************************************************ +// +// Kokkos +// Manycore Performance-Portable Multidimensional Arrays +// +// Copyright (2012) Sandia Corporation +// +// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +// the U.S. Government retains certain rights in this software. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// 1. Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// 3. Neither the name of the Corporation nor the names of the +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Questions? Contact H. Carter Edwards (hcedwar@sandia.gov) +// +// ************************************************************************ +//@HEADER +*/ + +#ifndef KOKKOS_STATICCRSGRAPH_HPP +#define KOKKOS_STATICCRSGRAPH_HPP + +#include +#include + +#include + +namespace Kokkos { + +/// \class StaticCrsGraph +/// \brief Compressed row storage array. +/// +/// \tparam DataType The type of stored entries. If a StaticCrsGraph is +/// used as the graph of a sparse matrix, then this is usually an +/// integer type, the type of the column indices in the sparse +/// matrix. +/// +/// \tparam Arg1Type The second template parameter, corresponding +/// either to the Device type (if there are no more template +/// parameters) or to the Layout type (if there is at least one more +/// template parameter). +/// +/// \tparam Arg2Type The third template parameter, which if provided +/// corresponds to the Device type. +/// +/// \tparam SizeType The type of row offsets. Usually the default +/// parameter suffices. However, setting a nondefault value is +/// necessary in some cases, for example, if you want to have a +/// sparse matrices with dimensions (and therefore column indices) +/// that fit in \c int, but want to store more than INT_MAX +/// entries in the sparse matrix. +/// +/// A row has a range of entries: +///
    +///
  • row_map[i0] <= entry < row_map[i0+1]
  • +///
  • 0 <= i1 < row_map[i0+1] - row_map[i0]
  • +///
  • entries( entry , i2 , i3 , ... );
  • +///
  • entries( row_map[i0] + i1 , i2 , i3 , ... );
  • +///
+template< class DataType, + class Arg1Type, + class Arg2Type = void, + typename SizeType = typename ViewTraits::size_type> +class StaticCrsGraph { +private: + typedef ViewTraits traits; + +public: + typedef DataType data_type; + typedef typename traits::array_layout array_layout; + typedef typename traits::device_type device_type; + typedef SizeType size_type; + + typedef StaticCrsGraph< DataType , Arg1Type , Arg2Type , SizeType > staticcrsgraph_type; + typedef StaticCrsGraph< DataType , array_layout , typename traits::host_mirror_space , SizeType > HostMirror; + typedef View< const size_type* , array_layout, device_type > row_map_type; + typedef View< DataType* , array_layout, device_type > entries_type; + + entries_type entries; + row_map_type row_map; + + //! Construct an empty view. + StaticCrsGraph () : entries(), row_map() {} + + //! Copy constructor (shallow copy). + StaticCrsGraph (const StaticCrsGraph& rhs) : entries (rhs.entries), row_map (rhs.row_map) + {} + + template + StaticCrsGraph (const EntriesType& entries_,const RowMapType& row_map_) : entries (entries_), row_map (row_map_) + {} + + /** \brief Assign to a view of the rhs array. + * If the old view is the last view + * then allocated memory is deallocated. + */ + StaticCrsGraph& operator= (const StaticCrsGraph& rhs) { + entries = rhs.entries; + row_map = rhs.row_map; + return *this; + } + + /** \brief Destroy this view of the array. + * If the last view then allocated memory is deallocated. + */ + ~StaticCrsGraph() {} + + size_t numRows() const { + return row_map.dimension_0()>0?row_map.dimension_0()-1:0; + } + +}; + +//---------------------------------------------------------------------------- + +template< class StaticCrsGraphType , class InputSizeType > +typename StaticCrsGraphType::staticcrsgraph_type +create_staticcrsgraph( const std::string & label , + const std::vector< InputSizeType > & input ); + +template< class StaticCrsGraphType , class InputSizeType > +typename StaticCrsGraphType::staticcrsgraph_type +create_staticcrsgraph( const std::string & label , + const std::vector< std::vector< InputSizeType > > & input ); + +//---------------------------------------------------------------------------- + +template< class DataType , + class Arg1Type , + class Arg2Type , + typename SizeType > +typename StaticCrsGraph< DataType , Arg1Type , Arg2Type , SizeType >::HostMirror +create_mirror_view( const StaticCrsGraph & input ); + +template< class DataType , + class Arg1Type , + class Arg2Type , + typename SizeType > +typename StaticCrsGraph< DataType , Arg1Type , Arg2Type , SizeType >::HostMirror +create_mirror( const StaticCrsGraph & input ); + +} // namespace Kokkos + +//---------------------------------------------------------------------------- +//---------------------------------------------------------------------------- + +#include + +//---------------------------------------------------------------------------- +//---------------------------------------------------------------------------- + +namespace Kokkos { +namespace Impl { + +template< class GraphType > +struct StaticCrsGraphMaximumEntry { + + typedef typename GraphType::device_type device_type ; + typedef typename GraphType::data_type value_type ; + + const typename GraphType::entries_type entries ; + + StaticCrsGraphMaximumEntry( const GraphType & graph ) : entries( graph.entries ) {} + + KOKKOS_INLINE_FUNCTION + void operator()( const unsigned i , value_type & update ) const + { if ( update < entries(i) ) update = entries(i); } + + KOKKOS_INLINE_FUNCTION + void init( value_type & update ) const + { update = 0 ; } + + KOKKOS_INLINE_FUNCTION + void join( volatile value_type & update , + volatile const value_type & input ) const + { if ( update < input ) update = input ; } +}; + +} + +template< class DataType, class Arg1Type, class Arg2Type, typename SizeType > +DataType maximum_entry( const StaticCrsGraph< DataType , Arg1Type , Arg2Type , SizeType > & graph ) +{ + typedef StaticCrsGraph GraphType ; + typedef Impl::StaticCrsGraphMaximumEntry< GraphType > FunctorType ; + + DataType result = 0 ; + Kokkos::parallel_reduce( graph.entries.dimension_0(), + FunctorType(graph), result ); + return result ; +} + +} // namespace Kokkos + +//---------------------------------------------------------------------------- +//---------------------------------------------------------------------------- + +#endif /* #ifndef KOKKOS_CRSARRAY_HPP */ + diff --git a/lib/kokkos/containers/src/Kokkos_UnorderedMap.hpp b/lib/kokkos/containers/src/Kokkos_UnorderedMap.hpp new file mode 100755 index 0000000000..ccf25f53d6 --- /dev/null +++ b/lib/kokkos/containers/src/Kokkos_UnorderedMap.hpp @@ -0,0 +1,848 @@ +/* +//@HEADER +// ************************************************************************ +// +// Kokkos: Manycore Performance-Portable Multidimensional Arrays +// Copyright (2012) Sandia Corporation +// +// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +// the U.S. Government retains certain rights in this software. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// 1. Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// 3. Neither the name of the Corporation nor the names of the +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Questions? Contact H. Carter Edwards (hcedwar@sandia.gov) +// +// ************************************************************************ +//@HEADER +*/ + +/// \file Kokkos_UnorderedMap.hpp +/// \brief Declaration and definition of Kokkos::UnorderedMap. +/// +/// This header file declares and defines Kokkos::UnorderedMap and its +/// related nonmember functions. + +#ifndef KOKKOS_UNORDERED_MAP_HPP +#define KOKKOS_UNORDERED_MAP_HPP + +#include +#include + +#include + +#include +#include + + +#include + +#include +#include + + +namespace Kokkos { + +enum { UnorderedMapInvalidIndex = ~0u }; + +/// \brief First element of the return value of UnorderedMap::insert(). +/// +/// Inserting an element into an UnorderedMap is not guaranteed to +/// succeed. There are three possible conditions: +///
    +///
  1. INSERT_FAILED: The insert failed. This usually +/// means that the UnorderedMap ran out of space.
  2. +///
  3. INSERT_SUCCESS: The insert succeeded, and the key +/// did not exist in the table before.
  4. +///
  5. INSERT_EXISTING: The insert succeeded, and the key +/// did exist in the table before. The new value was +/// ignored and the old value was left in place.
  6. +///
+ +class UnorderedMapInsertResult +{ +private: + enum Status{ + SUCCESS = 1u << 31 + , EXISTING = 1u << 30 + , FREED_EXISTING = 1u << 29 + , LIST_LENGTH_MASK = ~(SUCCESS | EXISTING | FREED_EXISTING) + }; + +public: + /// Did the map successful insert the key/value pair + KOKKOS_FORCEINLINE_FUNCTION + bool success() const { return (m_status & SUCCESS); } + + /// Was the key already present in the map + KOKKOS_FORCEINLINE_FUNCTION + bool existing() const { return (m_status & EXISTING); } + + /// Did the map fail to insert the key due to insufficent capacity + KOKKOS_FORCEINLINE_FUNCTION + bool failed() const { return m_index == UnorderedMapInvalidIndex; } + + /// Did the map lose a race condition to insert a dupulicate key/value pair + /// where an index was claimed that needed to be released + KOKKOS_FORCEINLINE_FUNCTION + bool freed_existing() const { return (m_status & FREED_EXISTING); } + + /// How many iterations through the insert loop did it take before the + /// map returned + KOKKOS_FORCEINLINE_FUNCTION + uint32_t list_position() const { return (m_status & LIST_LENGTH_MASK); } + + /// Index where the key can be found as long as the insert did not fail + KOKKOS_FORCEINLINE_FUNCTION + uint32_t index() const { return m_index; } + + KOKKOS_FORCEINLINE_FUNCTION + UnorderedMapInsertResult() + : m_index(UnorderedMapInvalidIndex) + , m_status(0) + {} + + KOKKOS_FORCEINLINE_FUNCTION + void increment_list_position() + { + m_status += (list_position() < LIST_LENGTH_MASK) ? 1u : 0u; + } + + KOKKOS_FORCEINLINE_FUNCTION + void set_existing(uint32_t i, bool arg_freed_existing) + { + m_index = i; + m_status = EXISTING | (arg_freed_existing ? FREED_EXISTING : 0u) | list_position(); + } + + KOKKOS_FORCEINLINE_FUNCTION + void set_success(uint32_t i) + { + m_index = i; + m_status = SUCCESS | list_position(); + } + +private: + uint32_t m_index; + uint32_t m_status; +}; + +/// \class UnorderedMap +/// \brief Thread-safe, performance-portable lookup table. +/// +/// This class provides a lookup table. In terms of functionality, +/// this class compares to std::unordered_map (new in C++11). +/// "Unordered" means that keys are not stored in any particular +/// order, unlike (for example) std::map. "Thread-safe" means that +/// lookups, insertion, and deletion are safe to call by multiple +/// threads in parallel. "Performance-portable" means that parallel +/// performance of these operations is reasonable, on multiple +/// hardware platforms. Platforms on which performance has been +/// tested include conventional Intel x86 multicore processors, Intel +/// Xeon Phi ("MIC"), and NVIDIA GPUs. +/// +/// Parallel performance portability entails design decisions that +/// might differ from one's expectation for a sequential interface. +/// This particularly affects insertion of single elements. In an +/// interface intended for sequential use, insertion might reallocate +/// memory if the original allocation did not suffice to hold the new +/// element. In this class, insertion does not reallocate +/// memory. This means that it might fail. insert() returns an enum +/// which indicates whether the insert failed. There are three +/// possible conditions: +///
    +///
  1. INSERT_FAILED: The insert failed. This usually +/// means that the UnorderedMap ran out of space.
  2. +///
  3. INSERT_SUCCESS: The insert succeeded, and the key +/// did not exist in the table before.
  4. +///
  5. INSERT_EXISTING: The insert succeeded, and the key +/// did exist in the table before. The new value was +/// ignored and the old value was left in place.
  6. +///
+/// +/// \tparam Key Type of keys of the lookup table. If \c const, users +/// are not allowed to add or remove keys, though they are allowed +/// to change values. In that case, the implementation may make +/// optimizations specific to the Device. For example, if +/// Device is \c Cuda, it may use texture fetches to access +/// keys. +/// +/// \tparam Value Type of values stored in the lookup table. You may use +/// \c void here, in which case the table will be a set of keys. If +/// \c const, users are not allowed to change entries. +/// In that case, the implementation may make +/// optimizations specific to the \c Device, such as using texture +/// fetches to access values. +/// +/// \tparam Device The Kokkos Device type. +/// +/// \tparam Hasher Definition of the hash function for instances of +/// Key. The default will calculate a bitwise hash. +/// +/// \tparam EqualTo Definition of the equality function for instances of +/// Key. The default will do a bitwise equality comparison. +/// +template < typename Key + , typename Value + , typename Device = Kokkos::DefaultExecutionSpace + , typename Hasher = pod_hash::type> + , typename EqualTo = pod_equal_to::type> + > +class UnorderedMap +{ +private: + typedef typename ViewTraits::host_mirror_space host_mirror_space ; +public: + //! \name Public types and constants + //@{ + + //key_types + typedef Key declared_key_type; + typedef typename Impl::remove_const::type key_type; + typedef typename Impl::add_const::type const_key_type; + + //value_types + typedef Value declared_value_type; + typedef typename Impl::remove_const::type value_type; + typedef typename Impl::add_const::type const_value_type; + + typedef Device device_type; + typedef Hasher hasher_type; + typedef EqualTo equal_to_type; + typedef uint32_t size_type; + + //map_types + typedef UnorderedMap declared_map_type; + typedef UnorderedMap insertable_map_type; + typedef UnorderedMap modifiable_map_type; + typedef UnorderedMap const_map_type; + + static const bool is_set = Impl::is_same::value; + static const bool has_const_key = Impl::is_same::value; + static const bool has_const_value = is_set || Impl::is_same::value; + + static const bool is_insertable_map = !has_const_key && (is_set || !has_const_value); + static const bool is_modifiable_map = has_const_key && !has_const_value; + static const bool is_const_map = has_const_key && has_const_value; + + + typedef UnorderedMapInsertResult insert_result; + + typedef UnorderedMap HostMirror; + + typedef Impl::UnorderedMapHistogram histogram_type; + + //@} + +private: + enum { invalid_index = ~static_cast(0) }; + + typedef typename Impl::if_c< is_set, int, declared_value_type>::type impl_value_type; + + typedef typename Impl::if_c< is_insertable_map + , View< key_type *, device_type> + , View< const key_type *, device_type, MemoryTraits > + >::type key_type_view; + + typedef typename Impl::if_c< is_insertable_map || is_modifiable_map + , View< impl_value_type *, device_type> + , View< const impl_value_type *, device_type, MemoryTraits > + >::type value_type_view; + + typedef typename Impl::if_c< is_insertable_map + , View< size_type *, device_type> + , View< const size_type *, device_type, MemoryTraits > + >::type size_type_view; + + typedef typename Impl::if_c< is_insertable_map + , Bitset< device_type > + , ConstBitset< device_type> + >::type bitset_type; + + enum { modified_idx = 0, erasable_idx = 1, failed_insert_idx = 2 }; + enum { num_scalars = 3 }; + typedef View< int[num_scalars], LayoutLeft, device_type> scalars_view; + +public: + //! \name Public member functions + //@{ + + UnorderedMap() + : m_bounded_insert() + , m_hasher() + , m_equal_to() + , m_size() + , m_available_indexes() + , m_hash_lists() + , m_next_index() + , m_keys() + , m_values() + , m_scalars() + {} + + /// \brief Constructor + /// + /// \param capacity_hint [in] Initial guess of how many unique keys will be inserted into the map + /// \param hash [in] Hasher function for \c Key instances. The + /// default value usually suffices. + UnorderedMap( size_type capacity_hint, hasher_type hasher = hasher_type(), equal_to_type equal_to = equal_to_type() ) + : m_bounded_insert(true) + , m_hasher(hasher) + , m_equal_to(equal_to) + , m_size() + , m_available_indexes(calculate_capacity(capacity_hint)) + , m_hash_lists(ViewAllocateWithoutInitializing("UnorderedMap hash list"), Impl::find_hash_size(capacity())) + , m_next_index(ViewAllocateWithoutInitializing("UnorderedMap next index"), capacity()+1) // +1 so that the *_at functions can always return a valid reference + , m_keys("UnorderedMap keys",capacity()+1) + , m_values("UnorderedMap values",(is_set? 1 : capacity()+1)) + , m_scalars("UnorderedMap scalars") + { + if (!is_insertable_map) { + throw std::runtime_error("Cannot construct a non-insertable (i.e. const key_type) unordered_map"); + } + + Kokkos::deep_copy(m_hash_lists, invalid_index); + Kokkos::deep_copy(m_next_index, invalid_index); + } + + void reset_failed_insert_flag() + { + reset_flag(failed_insert_idx); + } + + histogram_type get_histogram() + { + return histogram_type(*this); + } + + //! Clear all entries in the table. + void clear() + { + m_bounded_insert = true; + + if (capacity() == 0) return; + + m_available_indexes.clear(); + + Kokkos::deep_copy(m_hash_lists, invalid_index); + Kokkos::deep_copy(m_next_index, invalid_index); + { + const key_type tmp = key_type(); + Kokkos::deep_copy(m_keys,tmp); + } + if (is_set){ + const impl_value_type tmp = impl_value_type(); + Kokkos::deep_copy(m_values,tmp); + } + { + Kokkos::deep_copy(m_scalars, 0); + } + } + + /// \brief Change the capacity of the the map + /// + /// If there are no failed inserts the current size of the map will + /// be used as a lower bound for the input capacity. + /// If the map is not empty and does not have failed inserts + /// and the capacity changes then the current data is copied + /// into the resized / rehashed map. + /// + /// This is not a device function; it may not be + /// called in a parallel kernel. + bool rehash(size_type requested_capacity = 0) + { + const bool bounded_insert = (capacity() == 0) || (size() == 0u); + return rehash(requested_capacity, bounded_insert ); + } + + bool rehash(size_type requested_capacity, bool bounded_insert) + { + if(!is_insertable_map) return false; + + const size_type curr_size = size(); + requested_capacity = (requested_capacity < curr_size) ? curr_size : requested_capacity; + + insertable_map_type tmp(requested_capacity, m_hasher, m_equal_to); + + if (curr_size) { + tmp.m_bounded_insert = false; + Impl::UnorderedMapRehash f(tmp,*this); + f.apply(); + } + tmp.m_bounded_insert = bounded_insert; + + *this = tmp; + + return true; + } + + /// \brief The number of entries in the table. + /// + /// This method has undefined behavior when erasable() is true. + /// + /// Note that this is not a device function; it cannot be called in + /// a parallel kernel. The value is not stored as a variable; it + /// must be computed. + size_type size() const + { + if( capacity() == 0u ) return 0u; + if (modified()) { + m_size = m_available_indexes.count(); + reset_flag(modified_idx); + } + return m_size; + } + + /// \brief The current number of failed insert() calls. + /// + /// This is not a device function; it may not be + /// called in a parallel kernel. The value is not stored as a + /// variable; it must be computed. + bool failed_insert() const + { + return get_flag(failed_insert_idx); + } + + bool erasable() const + { + return is_insertable_map ? get_flag(erasable_idx) : false; + } + + bool begin_erase() + { + bool result = !erasable(); + if (is_insertable_map && result) { + device_type::fence(); + set_flag(erasable_idx); + device_type::fence(); + } + return result; + } + + bool end_erase() + { + bool result = erasable(); + if (is_insertable_map && result) { + device_type::fence(); + Impl::UnorderedMapErase f(*this); + f.apply(); + device_type::fence(); + reset_flag(erasable_idx); + } + return result; + } + + /// \brief The maximum number of entries that the table can hold. + /// + /// This is a device function; it may be called in a parallel + /// kernel. + KOKKOS_FORCEINLINE_FUNCTION + size_type capacity() const + { return m_available_indexes.size(); } + + /// \brief The number of hash table "buckets." + /// + /// This is different than the number of entries that the table can + /// hold. Each key hashes to an index in [0, hash_capacity() - 1]. + /// That index can hold zero or more entries. This class decides + /// what hash_capacity() should be, given the user's upper bound on + /// the number of entries the table must be able to hold. + /// + /// This is a device function; it may be called in a parallel + /// kernel. + KOKKOS_INLINE_FUNCTION + size_type hash_capacity() const + { return m_hash_lists.size(); } + + //--------------------------------------------------------------------------- + //--------------------------------------------------------------------------- + + + /// This is a device function; it may be called in a parallel + /// kernel. As discussed in the class documentation, it need not + /// succeed. The return value tells you if it did. + /// + /// \param k [in] The key to attempt to insert. + /// \param v [in] The corresponding value to attempt to insert. If + /// using this class as a set (with Value = void), then you need not + /// provide this value. + KOKKOS_INLINE_FUNCTION + insert_result insert(key_type const& k, impl_value_type const&v = impl_value_type()) const + { + insert_result result; + + if ( !is_insertable_map || capacity() == 0u || m_scalars((int)erasable_idx) ) { + return result; + } + + if ( !m_scalars((int)modified_idx) ) { + m_scalars((int)modified_idx) = true; + } + + int volatile & failed_insert_ref = m_scalars((int)failed_insert_idx) ; + + const size_type hash_value = m_hasher(k); + const size_type hash_list = hash_value % m_hash_lists.size(); + + size_type * curr_ptr = & m_hash_lists[ hash_list ]; + size_type new_index = invalid_index ; + + // Force integer multiply to long + size_type index_hint = static_cast( (static_cast(hash_list) * capacity()) / m_hash_lists.size()); + + size_type find_attempts = 0; + + enum { bounded_find_attempts = 32u }; + const size_type max_attempts = (m_bounded_insert && (bounded_find_attempts < m_available_indexes.max_hint()) ) ? + bounded_find_attempts : + m_available_indexes.max_hint(); + + bool not_done = true ; + +#if defined( __MIC__ ) + #pragma noprefetch +#endif + while ( not_done ) { + + // Continue searching the unordered list for this key, + // list will only be appended during insert phase. + // Need volatile_load as other threads may be appending. + size_type curr = volatile_load(curr_ptr); + + KOKKOS_NONTEMPORAL_PREFETCH_LOAD(&m_keys[curr != invalid_index ? curr : 0]); +#if defined( __MIC__ ) + #pragma noprefetch +#endif + while ( curr != invalid_index && ! m_equal_to( volatile_load(&m_keys[curr]), k) ) { + result.increment_list_position(); + index_hint = curr; + curr_ptr = &m_next_index[curr]; + curr = volatile_load(curr_ptr); + KOKKOS_NONTEMPORAL_PREFETCH_LOAD(&m_keys[curr != invalid_index ? curr : 0]); + } + + //------------------------------------------------------------ + // If key already present then return that index. + if ( curr != invalid_index ) { + + const bool free_existing = new_index != invalid_index; + if ( free_existing ) { + // Previously claimed an unused entry that was not inserted. + // Release this unused entry immediately. + if (!m_available_indexes.reset(new_index) ) { + printf("Unable to free existing\n"); + } + + } + + result.set_existing(curr, free_existing); + not_done = false ; + } + //------------------------------------------------------------ + // Key is not currently in the map. + // If the thread has claimed an entry try to insert now. + else { + + //------------------------------------------------------------ + // If have not already claimed an unused entry then do so now. + if (new_index == invalid_index) { + + bool found = false; + // use the hash_list as the flag for the search direction + Kokkos::tie(found, index_hint) = m_available_indexes.find_any_unset_near( index_hint, hash_list ); + + // found and index and this thread set it + if ( !found && ++find_attempts >= max_attempts ) { + failed_insert_ref = true; + not_done = false ; + } + else if (m_available_indexes.set(index_hint) ) { + new_index = index_hint; + // Set key and value + KOKKOS_NONTEMPORAL_PREFETCH_STORE(&m_keys[new_index]); + m_keys[new_index] = k ; + + if (!is_set) { + KOKKOS_NONTEMPORAL_PREFETCH_STORE(&m_values[new_index]); + m_values[new_index] = v ; + } + + // Do not proceed until key and value are updated in global memory + memory_fence(); + } + } + else if (failed_insert_ref) { + not_done = false; + } + + // Attempt to append claimed entry into the list. + // Another thread may also be trying to append the same list so protect with atomic. + if ( new_index != invalid_index && + curr == atomic_compare_exchange(curr_ptr, static_cast(invalid_index), new_index) ) { + // Succeeded in appending + result.set_success(new_index); + not_done = false ; + } + } + } // while ( not_done ) + + return result ; + } + + KOKKOS_INLINE_FUNCTION + bool erase(key_type const& k) const + { + bool result = false; + + if(is_insertable_map && 0u < capacity() && m_scalars((int)erasable_idx)) { + + if ( ! m_scalars((int)modified_idx) ) { + m_scalars((int)modified_idx) = true; + } + + size_type index = find(k); + if (valid_at(index)) { + m_available_indexes.reset(index); + result = true; + } + } + + return result; + } + + /// \brief Find the given key \c k, if it exists in the table. + /// + /// \return If the key exists in the table, the index of the + /// value corresponding to that key; otherwise, an invalid index. + /// + /// This is a device function; it may be called in a parallel + /// kernel. + KOKKOS_INLINE_FUNCTION + size_type find( const key_type & k) const + { + size_type curr = 0u < capacity() ? m_hash_lists( m_hasher(k) % m_hash_lists.size() ) : invalid_index ; + + KOKKOS_NONTEMPORAL_PREFETCH_LOAD(&m_keys[curr != invalid_index ? curr : 0]); + while (curr != invalid_index && !m_equal_to( m_keys[curr], k) ) { + KOKKOS_NONTEMPORAL_PREFETCH_LOAD(&m_keys[curr != invalid_index ? curr : 0]); + curr = m_next_index[curr]; + } + + return curr; + } + + /// \brief Does the key exist in the map + /// + /// This is a device function; it may be called in a parallel + /// kernel. + KOKKOS_INLINE_FUNCTION + bool exists( const key_type & k) const + { + return valid_at(find(k)); + } + + + /// \brief Get the value with \c i as its direct index. + /// + /// \param i [in] Index directly into the array of entries. + /// + /// This is a device function; it may be called in a parallel + /// kernel. + /// + /// 'const value_type' via Cuda texture fetch must return by value. + KOKKOS_FORCEINLINE_FUNCTION + typename Impl::if_c< (is_set || has_const_value), impl_value_type, impl_value_type &>::type + value_at(size_type i) const + { + return m_values[ is_set ? 0 : (i < capacity() ? i : capacity()) ]; + } + + /// \brief Get the key with \c i as its direct index. + /// + /// \param i [in] Index directly into the array of entries. + /// + /// This is a device function; it may be called in a parallel + /// kernel. + KOKKOS_FORCEINLINE_FUNCTION + key_type key_at(size_type i) const + { + return m_keys[ i < capacity() ? i : capacity() ]; + } + + KOKKOS_FORCEINLINE_FUNCTION + bool valid_at(size_type i) const + { + return m_available_indexes.test(i); + } + + template + UnorderedMap( UnorderedMap const& src, + typename Impl::enable_if< Impl::UnorderedMapCanAssign::value,int>::type = 0 + ) + : m_bounded_insert(src.m_bounded_insert) + , m_hasher(src.m_hasher) + , m_equal_to(src.m_equal_to) + , m_size(src.m_size) + , m_available_indexes(src.m_available_indexes) + , m_hash_lists(src.m_hash_lists) + , m_next_index(src.m_next_index) + , m_keys(src.m_keys) + , m_values(src.m_values) + , m_scalars(src.m_scalars) + {} + + + template + typename Impl::enable_if< Impl::UnorderedMapCanAssign::value + ,declared_map_type & >::type + operator=( UnorderedMap const& src) + { + m_bounded_insert = src.m_bounded_insert; + m_hasher = src.m_hasher; + m_equal_to = src.m_equal_to; + m_size = src.m_size; + m_available_indexes = src.m_available_indexes; + m_hash_lists = src.m_hash_lists; + m_next_index = src.m_next_index; + m_keys = src.m_keys; + m_values = src.m_values; + m_scalars = src.m_scalars; + return *this; + } + + template + typename Impl::enable_if< Impl::is_same< typename Impl::remove_const::type, key_type>::value && + Impl::is_same< typename Impl::remove_const::type, value_type>::value + >::type + create_copy_view( UnorderedMap const& src) + { + if (m_hash_lists.ptr_on_device() != src.m_hash_lists.ptr_on_device()) { + + insertable_map_type tmp; + + tmp.m_bounded_insert = src.m_bounded_insert; + tmp.m_hasher = src.m_hasher; + tmp.m_equal_to = src.m_equal_to; + tmp.m_size = src.size(); + tmp.m_available_indexes = bitset_type( src.capacity() ); + tmp.m_hash_lists = size_type_view( ViewAllocateWithoutInitializing("UnorderedMap hash list"), src.m_hash_lists.size() ); + tmp.m_next_index = size_type_view( ViewAllocateWithoutInitializing("UnorderedMap next index"), src.m_next_index.size() ); + tmp.m_keys = key_type_view( ViewAllocateWithoutInitializing("UnorderedMap keys"), src.m_keys.size() ); + tmp.m_values = value_type_view( ViewAllocateWithoutInitializing("UnorderedMap values"), src.m_values.size() ); + tmp.m_scalars = scalars_view("UnorderedMap scalars"); + + Kokkos::deep_copy(tmp.m_available_indexes, src.m_available_indexes); + + typedef Kokkos::Impl::DeepCopy< typename device_type::memory_space, typename SDevice::memory_space > raw_deep_copy; + + raw_deep_copy(tmp.m_hash_lists.ptr_on_device(), src.m_hash_lists.ptr_on_device(), sizeof(size_type)*src.m_hash_lists.size()); + raw_deep_copy(tmp.m_next_index.ptr_on_device(), src.m_next_index.ptr_on_device(), sizeof(size_type)*src.m_next_index.size()); + raw_deep_copy(tmp.m_keys.ptr_on_device(), src.m_keys.ptr_on_device(), sizeof(key_type)*src.m_keys.size()); + if (!is_set) { + raw_deep_copy(tmp.m_values.ptr_on_device(), src.m_values.ptr_on_device(), sizeof(impl_value_type)*src.m_values.size()); + } + raw_deep_copy(tmp.m_scalars.ptr_on_device(), src.m_scalars.ptr_on_device(), sizeof(int)*num_scalars ); + + *this = tmp; + } + } + + //@} +private: // private member functions + + bool modified() const + { + return get_flag(modified_idx); + } + + void set_flag(int flag) const + { + typedef Kokkos::Impl::DeepCopy< typename device_type::memory_space, Kokkos::HostSpace > raw_deep_copy; + const int true_ = true; + raw_deep_copy(m_scalars.ptr_on_device() + flag, &true_, sizeof(int)); + } + + void reset_flag(int flag) const + { + typedef Kokkos::Impl::DeepCopy< typename device_type::memory_space, Kokkos::HostSpace > raw_deep_copy; + const int false_ = false; + raw_deep_copy(m_scalars.ptr_on_device() + flag, &false_, sizeof(int)); + } + + bool get_flag(int flag) const + { + typedef Kokkos::Impl::DeepCopy< Kokkos::HostSpace, typename device_type::memory_space > raw_deep_copy; + int result = false; + raw_deep_copy(&result, m_scalars.ptr_on_device() + flag, sizeof(int)); + return result; + } + + static uint32_t calculate_capacity(uint32_t capacity_hint) + { + // increase by 16% and round to nears multiple of 128 + return capacity_hint ? ((static_cast(7ull*capacity_hint/6u) + 127u)/128u)*128u : 128u; + } + +private: // private members + bool m_bounded_insert; + hasher_type m_hasher; + equal_to_type m_equal_to; + mutable size_type m_size; + bitset_type m_available_indexes; + size_type_view m_hash_lists; + size_type_view m_next_index; + key_type_view m_keys; + value_type_view m_values; + scalars_view m_scalars; + + template + friend class UnorderedMap; + + template + friend struct Impl::UnorderedMapErase; + + template + friend struct Impl::UnorderedMapHistogram; + + template + friend struct Impl::UnorderedMapPrint; +}; + +// Specialization of deep_copy for two UnorderedMap objects. +template < typename DKey, typename DT, typename DDevice + , typename SKey, typename ST, typename SDevice + , typename Hasher, typename EqualTo > +inline void deep_copy( UnorderedMap & dst + , const UnorderedMap & src ) +{ + dst.create_copy_view(src); +} + + +} // namespace Kokkos + +#endif //KOKKOS_UNORDERED_MAP_HPP diff --git a/lib/kokkos/containers/src/Kokkos_Vector.hpp b/lib/kokkos/containers/src/Kokkos_Vector.hpp new file mode 100755 index 0000000000..ded6512e62 --- /dev/null +++ b/lib/kokkos/containers/src/Kokkos_Vector.hpp @@ -0,0 +1,282 @@ +/* +//@HEADER +// ************************************************************************ +// +// Kokkos +// Manycore Performance-Portable Multidimensional Arrays +// +// Copyright (2012) Sandia Corporation +// +// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +// the U.S. Government retains certain rights in this software. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// 1. Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// 3. Neither the name of the Corporation nor the names of the +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Questions? Contact H. Carter Edwards (hcedwar@sandia.gov) +// +// ************************************************************************ +//@HEADER +*/ + +#ifndef KOKKOS_VECTOR_HPP +#define KOKKOS_VECTOR_HPP + +#include +#include + +/* Drop in replacement for std::vector based on Kokkos::DualView + * Most functions only work on the host (it will not compile if called from device kernel) + * + */ + namespace Kokkos { + +template +class vector : public DualView { +public: + typedef Device device_type; + typedef Scalar value_type; + typedef Scalar* pointer; + typedef const Scalar* const_pointer; + typedef Scalar* reference; + typedef const Scalar* const_reference; + typedef Scalar* iterator; + typedef const Scalar* const_iterator; + +private: + size_t _size; + typedef size_t size_type; + float _extra_storage; + typedef DualView DV; + + +public: + inline Scalar& operator() (int i) const {return DV::h_view(i);}; + inline Scalar& operator[] (int i) const {return DV::h_view(i);}; + + + /* Member functions which behave like std::vector functions */ + + vector():DV() { + _size = 0; + _extra_storage = 1.1; + DV::modified_host = 1; + }; + + + vector(int n, Scalar val=Scalar()):DualView("Vector",size_t(n*(1.1))) { + _size = n; + _extra_storage = 1.1; + DV::modified_host = 1; + + assign(n,val); + } + + + void resize(size_t n) { + if(n>=capacity()) + DV::resize(size_t (n*_extra_storage)); + _size = n; + } + + void resize(size_t n, const Scalar& val) { + assign(n,val); + } + + void assign (size_t n, const Scalar& val) { + + /* Resize if necessary (behavour of std:vector) */ + + if(n>capacity()) + DV::resize(size_t (n*_extra_storage)); + _size = n; + + /* Assign value either on host or on device */ + + if( DV::modified_host >= DV::modified_device ) { + set_functor_host f(DV::h_view,val); + parallel_for(n,f); + DV::t_host::device_type::fence(); + DV::modified_host++; + } else { + set_functor f(DV::d_view,val); + parallel_for(n,f); + DV::t_dev::device_type::fence(); + DV::modified_device++; + } + } + + void reserve(size_t n) { + DV::resize(size_t (n*_extra_storage)); + } + + void push_back(Scalar val) { + DV::modified_host++; + if(_size == capacity()) { + size_t new_size = _size*_extra_storage; + if(new_size == _size) new_size++; + DV::resize(new_size); + } + + DV::h_view(_size) = val; + _size++; + + }; + + void pop_back() { + _size--; + }; + + void clear() { + _size = 0; + } + + size_type size() const {return _size;}; + size_type max_size() const {return 2000000000;} + size_type capacity() const {return DV::capacity();}; + bool empty() const {return _size==0;}; + + iterator begin() const {return &DV::h_view(0);}; + + iterator end() const {return &DV::h_view(_size);}; + + + /* std::algorithms wich work originally with iterators, here they are implemented as member functions */ + + size_t + lower_bound (const size_t& start, + const size_t& theEnd, + const Scalar& comp_val) const + { + int lower = start; // FIXME (mfh 24 Apr 2014) narrowing conversion + int upper = _size > theEnd? theEnd : _size-1; // FIXME (mfh 24 Apr 2014) narrowing conversion + if (upper <= lower) { + return theEnd; + } + + Scalar lower_val = DV::h_view(lower); + Scalar upper_val = DV::h_view(upper); + size_t idx = (upper+lower)/2; + Scalar val = DV::h_view(idx); + if(val>upper_val) return upper; + if(vallower) { + if(comp_val>val) { + lower = ++idx; + } else { + upper = idx; + } + idx = (upper+lower)/2; + val = DV::h_view(idx); + } + return idx; + } + + bool is_sorted() { + for(int i=0;i<_size-1;i++) { + if(DV::h_view(i)>DV::h_view(i+1)) return false; + } + return true; + } + + iterator find(Scalar val) const { + if(_size == 0) return end(); + + int upper,lower,current; + current = _size/2; + upper = _size-1; + lower = 0; + + if((valDV::h_view(_size-1)) ) return end(); + + while(upper>lower) + { + if(val>DV::h_view(current)) lower = current+1; + else upper = current; + current = (upper+lower)/2; + } + + if(val==DV::h_view(current)) return &DV::h_view(current); + else return end(); + } + + /* Additional functions for data management */ + + void device_to_host(){ + deep_copy(DV::h_view,DV::d_view); + } + void host_to_device() const { + deep_copy(DV::d_view,DV::h_view); + } + + void on_host() { + DV::modified_host = DV::modified_device + 1; + } + void on_device() { + DV::modified_device = DV::modified_host + 1; + } + + void set_overallocation(float extra) { + _extra_storage = 1.0 + extra; + } + + +public: + struct set_functor { + typedef typename DV::t_dev::device_type device_type; + typename DV::t_dev _data; + Scalar _val; + + set_functor(typename DV::t_dev data, Scalar val) : + _data(data),_val(val) {} + + KOKKOS_INLINE_FUNCTION + void operator() (const int &i) const { + _data(i) = _val; + } + }; + + struct set_functor_host { + typedef typename DV::t_host::device_type device_type; + typename DV::t_host _data; + Scalar _val; + + set_functor_host(typename DV::t_host data, Scalar val) : + _data(data),_val(val) {} + + KOKKOS_INLINE_FUNCTION + void operator() (const int &i) const { + _data(i) = _val; + } + }; + +}; + + +} +#endif diff --git a/lib/kokkos/containers/src/impl/Kokkos_Bitset_impl.hpp b/lib/kokkos/containers/src/impl/Kokkos_Bitset_impl.hpp new file mode 100755 index 0000000000..dde5bffdfd --- /dev/null +++ b/lib/kokkos/containers/src/impl/Kokkos_Bitset_impl.hpp @@ -0,0 +1,173 @@ +/* +//@HEADER +// ************************************************************************ +// +// Kokkos: Manycore Performance-Portable Multidimensional Arrays +// Copyright (2012) Sandia Corporation +// +// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +// the U.S. Government retains certain rights in this software. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// 1. Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// 3. Neither the name of the Corporation nor the names of the +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Questions? Contact H. Carter Edwards (hcedwar@sandia.gov) +// +// ************************************************************************ +//@HEADER +*/ + +#ifndef KOKKOS_BITSET_IMPL_HPP +#define KOKKOS_BITSET_IMPL_HPP + +#include +#include + +#include +#include +#include +#include + +namespace Kokkos { namespace Impl { + +KOKKOS_FORCEINLINE_FUNCTION +unsigned rotate_right(unsigned i, int r) +{ + enum { size = static_cast(sizeof(unsigned)*CHAR_BIT) }; + return r ? ((i >> r) | (i << (size-r))) : i ; +} + +KOKKOS_FORCEINLINE_FUNCTION +int bit_scan_forward(unsigned i) +{ +#if defined( __CUDA_ARCH__ ) + return __ffs(i) - 1; +#elif defined( __GNUC__ ) || defined( __GNUG__ ) + return __builtin_ffs(i) - 1; +#elif defined( __INTEL_COMPILER ) + return _bit_scan_forward(i); +#else + + unsigned t = 1u; + int r = 0; + while (i && (i & t == 0)) + { + t = t << 1; + ++r; + } + return r; +#endif +} + + +KOKKOS_FORCEINLINE_FUNCTION +int bit_scan_reverse(unsigned i) +{ + enum { shift = static_cast(sizeof(unsigned)*CHAR_BIT - 1) }; +#if defined( __CUDA_ARCH__ ) + return shift - __clz(i); +#elif defined( __GNUC__ ) || defined( __GNUG__ ) + return shift - __builtin_clz(i); +#elif defined( __INTEL_COMPILER ) + return _bit_scan_reverse(i); +#else + unsigned t = 1u << shift; + int r = 0; + while (i && (i & t == 0)) + { + t = t >> 1; + ++r; + } + return r; +#endif +} + + +// count the bits set +KOKKOS_FORCEINLINE_FUNCTION +int popcount(unsigned i) +{ +#if defined( __CUDA_ARCH__ ) + return __popc(i); +#elif defined( __GNUC__ ) || defined( __GNUG__ ) + return __builtin_popcount(i); +#elif defined ( __INTEL_COMPILER ) + return _popcnt32(i); +#else + // http://graphics.stanford.edu/~seander/bithacks.html#CountBitsSetNaive + i = i - ((i >> 1) & ~0u/3u); // temp + i = (i & ~0u/15u*3u) + ((i >> 2) & ~0u/15u*3u); // temp + i = (i + (i >> 4)) & ~0u/255u*15u; // temp + return (int)((i * (~0u/255u)) >> (sizeof(unsigned) - 1) * CHAR_BIT); // count +#endif +} + + +template +struct BitsetCount +{ + typedef Bitset bitset_type; + typedef typename bitset_type::device_type::execution_space device_type; + typedef typename bitset_type::size_type size_type; + typedef size_type value_type; + + bitset_type m_bitset; + + BitsetCount( bitset_type const& bitset) + : m_bitset(bitset) + {} + + size_type apply() const + { + size_type count = 0u; + parallel_reduce(m_bitset.m_blocks.size(), *this, count); + return count; + } + + KOKKOS_INLINE_FUNCTION + static void init( value_type & count) + { + count = 0u; + } + + KOKKOS_INLINE_FUNCTION + static void join( volatile value_type & count, const volatile size_type & incr ) + { + count += incr; + } + + KOKKOS_INLINE_FUNCTION + void operator()( size_type i, value_type & count) const + { + count += popcount(m_bitset.m_blocks[i]); + } +}; + +}} //Kokkos::Impl + +#endif // KOKKOS_BITSET_IMPL_HPP + diff --git a/lib/kokkos/containers/src/impl/Kokkos_Functional_impl.hpp b/lib/kokkos/containers/src/impl/Kokkos_Functional_impl.hpp new file mode 100755 index 0000000000..647024f48f --- /dev/null +++ b/lib/kokkos/containers/src/impl/Kokkos_Functional_impl.hpp @@ -0,0 +1,154 @@ +#ifndef KOKKOS_FUNCTIONAL_IMPL_HPP +#define KOKKOS_FUNCTIONAL_IMPL_HPP + +#include +#include + +namespace Kokkos { namespace Impl { + +// MurmurHash3 was written by Austin Appleby, and is placed in the public +// domain. The author hereby disclaims copyright to this source code. +KOKKOS_FORCEINLINE_FUNCTION +uint32_t getblock32 ( const uint8_t * p, int i ) +{ +// used to avoid aliasing error which could cause errors with +// forced inlining + return ((uint32_t)p[i*4+0]) + | ((uint32_t)p[i*4+1] << 8) + | ((uint32_t)p[i*4+2] << 16) + | ((uint32_t)p[i*4+3] << 24); +} + +KOKKOS_FORCEINLINE_FUNCTION +uint32_t rotl32 ( uint32_t x, int8_t r ) +{ return (x << r) | (x >> (32 - r)); } + +KOKKOS_FORCEINLINE_FUNCTION +uint32_t fmix32 ( uint32_t h ) +{ + h ^= h >> 16; + h *= 0x85ebca6b; + h ^= h >> 13; + h *= 0xc2b2ae35; + h ^= h >> 16; + + return h; +} + +KOKKOS_INLINE_FUNCTION +uint32_t MurmurHash3_x86_32 ( const void * key, int len, uint32_t seed ) +{ + const uint8_t * data = (const uint8_t*)key; + const int nblocks = len / 4; + + uint32_t h1 = seed; + + const uint32_t c1 = 0xcc9e2d51; + const uint32_t c2 = 0x1b873593; + + //---------- + // body + + for(int i=0; i +KOKKOS_FORCEINLINE_FUNCTION +bool bitwise_equal(T const * const a_ptr, T const * const b_ptr) +{ + typedef uint64_t KOKKOS_MAY_ALIAS T64; + typedef uint32_t KOKKOS_MAY_ALIAS T32; + typedef uint16_t KOKKOS_MAY_ALIAS T16; + typedef uint8_t KOKKOS_MAY_ALIAS T8; + + enum { + NUM_8 = sizeof(T), + NUM_16 = NUM_8 / 2, + NUM_32 = NUM_8 / 4, + NUM_64 = NUM_8 / 8 + }; + + union { + T const * const ptr; + T64 const * const ptr64; + T32 const * const ptr32; + T16 const * const ptr16; + T8 const * const ptr8; + } a = {a_ptr}, b = {b_ptr}; + + bool result = true; + + for (int i=0; i < NUM_64; ++i) { + result = result && a.ptr64[i] == b.ptr64[i]; + } + + if ( NUM_64*2 < NUM_32 ) { + result = result && a.ptr32[NUM_64*2] == b.ptr32[NUM_64*2]; + } + + if ( NUM_32*2 < NUM_16 ) { + result = result && a.ptr16[NUM_32*2] == b.ptr16[NUM_32*2]; + } + + if ( NUM_16*2 < NUM_8 ) { + result = result && a.ptr8[NUM_16*2] == b.ptr8[NUM_16*2]; + } + + return result; +} + + + +#undef KOKKOS_MAY_ALIAS + +}} // namespace Kokkos::Impl + +#endif //KOKKOS_FUNCTIONAL_IMPL_HPP diff --git a/lib/kokkos/containers/src/impl/Kokkos_StaticCrsGraph_factory.hpp b/lib/kokkos/containers/src/impl/Kokkos_StaticCrsGraph_factory.hpp new file mode 100755 index 0000000000..ddd091a457 --- /dev/null +++ b/lib/kokkos/containers/src/impl/Kokkos_StaticCrsGraph_factory.hpp @@ -0,0 +1,223 @@ +/* +//@HEADER +// ************************************************************************ +// +// Kokkos: Manycore Performance-Portable Multidimensional Arrays +// Copyright (2012) Sandia Corporation +// +// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +// the U.S. Government retains certain rights in this software. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// 1. Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// 3. Neither the name of the Corporation nor the names of the +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Questions? Contact H. Carter Edwards (hcedwar@sandia.gov) +// +// ************************************************************************ +//@HEADER +*/ + +#ifndef KOKKOS_IMPL_STATICCRSGRAPH_FACTORY_HPP +#define KOKKOS_IMPL_STATICCRSGRAPH_FACTORY_HPP + +//---------------------------------------------------------------------------- +//---------------------------------------------------------------------------- + +namespace Kokkos { + +template< class DataType , class Arg1Type , class Arg2Type , typename SizeType > +inline +typename StaticCrsGraph< DataType , Arg1Type , Arg2Type , SizeType >::HostMirror +create_mirror_view( const StaticCrsGraph & view , + typename Impl::enable_if< ViewTraits::is_hostspace >::type * = 0 ) +{ + return view ; +} + +template< class DataType , class Arg1Type , class Arg2Type , typename SizeType > +inline +typename StaticCrsGraph< DataType , Arg1Type , Arg2Type , SizeType >::HostMirror +create_mirror( const StaticCrsGraph & view ) +{ + // Force copy: + //typedef Impl::ViewAssignment< Impl::ViewDefault > alloc ; // unused + typedef StaticCrsGraph< DataType , Arg1Type , Arg2Type , SizeType > staticcrsgraph_type ; + + typename staticcrsgraph_type::HostMirror tmp ; + typename staticcrsgraph_type::row_map_type::HostMirror tmp_row_map = create_mirror( view.row_map); + + // Allocation to match: + tmp.row_map = tmp_row_map ; // Assignment of 'const' from 'non-const' + tmp.entries = create_mirror( view.entries ); + + + // Deep copy: + deep_copy( tmp_row_map , view.row_map ); + deep_copy( tmp.entries , view.entries ); + + return tmp ; +} + +template< class DataType , class Arg1Type , class Arg2Type , typename SizeType > +inline +typename StaticCrsGraph< DataType , Arg1Type , Arg2Type , SizeType >::HostMirror +create_mirror_view( const StaticCrsGraph & view , + typename Impl::enable_if< ! ViewTraits::is_hostspace >::type * = 0 ) +{ + return create_mirror( view ); +} +} // namespace Kokkos + +//---------------------------------------------------------------------------- +//---------------------------------------------------------------------------- + +namespace Kokkos { + +template< class StaticCrsGraphType , class InputSizeType > +inline +typename StaticCrsGraphType::staticcrsgraph_type +create_staticcrsgraph( const std::string & label , + const std::vector< InputSizeType > & input ) +{ + typedef StaticCrsGraphType output_type ; + //typedef std::vector< InputSizeType > input_type ; // unused + + typedef typename output_type::entries_type entries_type ; + + typedef View< typename output_type::size_type [] , + typename output_type::array_layout , + typename output_type::device_type > work_type ; + + output_type output ; + + // Create the row map: + + const size_t length = input.size(); + + { + work_type row_work( "tmp" , length + 1 ); + + typename work_type::HostMirror row_work_host = + create_mirror_view( row_work ); + + size_t sum = 0 ; + row_work_host[0] = 0 ; + for ( size_t i = 0 ; i < length ; ++i ) { + row_work_host[i+1] = sum += input[i]; + } + + deep_copy( row_work , row_work_host ); + + output.entries = entries_type( label , sum ); + output.row_map = row_work ; + } + + return output ; +} + +//---------------------------------------------------------------------------- + +template< class StaticCrsGraphType , class InputSizeType > +inline +typename StaticCrsGraphType::staticcrsgraph_type +create_staticcrsgraph( const std::string & label , + const std::vector< std::vector< InputSizeType > > & input ) +{ + typedef StaticCrsGraphType output_type ; + //typedef std::vector< std::vector< InputSizeType > > input_type ; // unused + typedef typename output_type::entries_type entries_type ; + //typedef typename output_type::size_type size_type ; // unused + + // mfh 14 Feb 2014: This function doesn't actually create instances + // of ok_rank, but it needs to declare the typedef in order to do + // the static "assert" (a compile-time check that the given shape + // has rank 1). In order to avoid a "declared but unused typedef" + // warning, we declare an empty instance of this type, with the + // usual "(void)" marker to avoid a compiler warning for the unused + // variable. + + typedef typename + Impl::assert_shape_is_rank_one< typename entries_type::shape_type >::type + ok_rank ; + { + ok_rank thing; + (void) thing; + } + + typedef View< typename output_type::size_type [] , + typename output_type::array_layout , + typename output_type::device_type > work_type ; + + output_type output ; + + // Create the row map: + + const size_t length = input.size(); + + { + work_type row_work( "tmp" , length + 1 ); + + typename work_type::HostMirror row_work_host = + create_mirror_view( row_work ); + + size_t sum = 0 ; + row_work_host[0] = 0 ; + for ( size_t i = 0 ; i < length ; ++i ) { + row_work_host[i+1] = sum += input[i].size(); + } + + deep_copy( row_work , row_work_host ); + + output.entries = entries_type( label , sum ); + output.row_map = row_work ; + } + + // Fill in the entries: + { + typename entries_type::HostMirror host_entries = + create_mirror_view( output.entries ); + + size_t sum = 0 ; + for ( size_t i = 0 ; i < length ; ++i ) { + for ( size_t j = 0 ; j < input[i].size() ; ++j , ++sum ) { + host_entries( sum ) = input[i][j] ; + } + } + + deep_copy( output.entries , host_entries ); + } + + return output ; +} + +} // namespace Kokkos + +//---------------------------------------------------------------------------- +//---------------------------------------------------------------------------- + +#endif /* #ifndef KOKKOS_IMPL_CRSARRAY_FACTORY_HPP */ + diff --git a/lib/kokkos/containers/src/impl/Kokkos_UnorderedMap_impl.cpp b/lib/kokkos/containers/src/impl/Kokkos_UnorderedMap_impl.cpp new file mode 100755 index 0000000000..150d3d893e --- /dev/null +++ b/lib/kokkos/containers/src/impl/Kokkos_UnorderedMap_impl.cpp @@ -0,0 +1,101 @@ +/* +//@HEADER +// ************************************************************************ +// +// Kokkos: Manycore Performance-Portable Multidimensional Arrays +// Copyright (2012) Sandia Corporation +// +// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +// the U.S. Government retains certain rights in this software. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// 1. Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// 3. Neither the name of the Corporation nor the names of the +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Questions? Contact H. Carter Edwards (hcedwar@sandia.gov) +// +// ************************************************************************ +//@HEADER +*/ + +#include + +namespace Kokkos { namespace Impl { + +uint32_t find_hash_size(uint32_t size) +{ + if (size == 0u) return 0u; + + // these primes try to preserve randomness of hash + static const uint32_t primes [] = { + 3, 7, 13, 23, 53, 97, 193, 389, 769, 1543 + , 2237, 2423, 2617, 2797, 2999, 3167, 3359, 3539 + , 3727, 3911, 4441 , 4787 , 5119 , 5471 , 5801 , 6143 , 6521 , 6827 + , 7177 , 7517 , 7853 , 8887 , 9587 , 10243 , 10937 , 11617 , 12289 + , 12967 , 13649 , 14341 , 15013 , 15727 + , 17749 , 19121 , 20479 , 21859 , 23209 , 24593 , 25939 , 27329 + , 28669 , 30047 , 31469 , 35507 , 38231 , 40961 , 43711 , 46439 + , 49157 , 51893 , 54617 , 57347 , 60077 , 62801 , 70583 , 75619 + , 80669 , 85703 , 90749 , 95783 , 100823 , 105871 , 110909 , 115963 + , 120997 , 126031 , 141157 , 151237 , 161323 , 171401 , 181499 , 191579 + , 201653 , 211741 , 221813 , 231893 , 241979 , 252079 + , 282311 , 302483 , 322649 , 342803 , 362969 , 383143 , 403301 , 423457 + , 443629 , 463787 , 483953 , 504121 , 564617 , 604949 , 645313 , 685609 + , 725939 , 766273 , 806609 , 846931 , 887261 , 927587 , 967919 , 1008239 + , 1123477 , 1198397 , 1273289 , 1348177 , 1423067 , 1497983 , 1572869 + , 1647761 , 1722667 , 1797581 , 1872461 , 1947359 , 2022253 + , 2246953 , 2396759 , 2546543 , 2696363 , 2846161 , 2995973 , 3145739 + , 3295541 , 3445357 , 3595117 , 3744941 , 3894707 , 4044503 + , 4493921 , 4793501 , 5093089 , 5392679 , 5692279 , 5991883 , 6291469 + , 6591059 , 6890641 , 7190243 , 7489829 , 7789447 , 8089033 + , 8987807 , 9586981 , 10186177 , 10785371 , 11384539 , 11983729 + , 12582917 , 13182109 , 13781291 , 14380469 , 14979667 , 15578861 + , 16178053 , 17895707 , 19014187 , 20132683 , 21251141 , 22369661 + , 23488103 , 24606583 , 25725083 , 26843549 , 27962027 , 29080529 + , 30198989 , 31317469 , 32435981 , 35791397 , 38028379 , 40265327 + , 42502283 , 44739259 , 46976221 , 49213237 , 51450131 , 53687099 + , 55924061 , 58161041 , 60397993 , 62634959 , 64871921 + , 71582857 , 76056727 , 80530643 , 85004567 , 89478503 , 93952427 + , 98426347 , 102900263 , 107374217 , 111848111 , 116322053 , 120795971 + , 125269877 , 129743807 , 143165587 , 152113427 , 161061283 , 170009141 + , 178956983 , 187904819 , 196852693 , 205800547 , 214748383 , 223696237 + , 232644089 , 241591943 , 250539763 , 259487603 , 268435399 + }; + + const uint32_t num_primes = sizeof(primes)/sizeof(uint32_t); + + uint32_t hsize = primes[num_primes-1] ; + for (uint32_t i = 0; i < num_primes; ++i) { + if (size <= primes[i]) { + hsize = primes[i]; + break; + } + } + return hsize; +} + +}} // namespace Kokkos::Impl + diff --git a/lib/kokkos/containers/src/impl/Kokkos_UnorderedMap_impl.hpp b/lib/kokkos/containers/src/impl/Kokkos_UnorderedMap_impl.hpp new file mode 100755 index 0000000000..b5c3304fba --- /dev/null +++ b/lib/kokkos/containers/src/impl/Kokkos_UnorderedMap_impl.hpp @@ -0,0 +1,297 @@ +/* +//@HEADER +// ************************************************************************ +// +// Kokkos: Manycore Performance-Portable Multidimensional Arrays +// Copyright (2012) Sandia Corporation +// +// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +// the U.S. Government retains certain rights in this software. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// 1. Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// 3. Neither the name of the Corporation nor the names of the +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Questions? Contact H. Carter Edwards (hcedwar@sandia.gov) +// +// ************************************************************************ +//@HEADER +*/ + +#ifndef KOKKOS_UNORDERED_MAP_IMPL_HPP +#define KOKKOS_UNORDERED_MAP_IMPL_HPP + +#include +#include + +#include +#include +#include +#include + +namespace Kokkos { namespace Impl { + +uint32_t find_hash_size( uint32_t size ); + +template +struct UnorderedMapRehash +{ + typedef Map map_type; + typedef typename map_type::const_map_type const_map_type; + typedef typename map_type::device_type device_type; + typedef typename map_type::size_type size_type; + + map_type m_dst; + const_map_type m_src; + + UnorderedMapRehash( map_type const& dst, const_map_type const& src) + : m_dst(dst), m_src(src) + {} + + void apply() const + { + parallel_for(m_src.capacity(), *this); + } + + KOKKOS_INLINE_FUNCTION + void operator()(size_type i) const + { + if ( m_src.valid_at(i) ) + m_dst.insert(m_src.key_at(i), m_src.value_at(i)); + } + +}; + +template +struct UnorderedMapErase +{ + typedef UMap map_type; + typedef typename map_type::device_type device_type; + typedef typename map_type::size_type size_type; + typedef typename map_type::key_type key_type; + typedef typename map_type::impl_value_type value_type; + + map_type m_map; + + UnorderedMapErase( map_type const& map) + : m_map(map) + {} + + void apply() const + { + parallel_for(m_map.m_hash_lists.size(), *this); + } + + KOKKOS_INLINE_FUNCTION + void operator()( size_type i ) const + { + const size_type invalid_index = map_type::invalid_index; + + size_type curr = m_map.m_hash_lists(i); + size_type next = invalid_index; + + // remove erased head of the linked-list + while (curr != invalid_index && !m_map.valid_at(curr)) { + next = m_map.m_next_index[curr]; + m_map.m_next_index[curr] = invalid_index; + m_map.m_keys[curr] = key_type(); + if (m_map.is_set) m_map.m_values[curr] = value_type(); + curr = next; + m_map.m_hash_lists(i) = next; + } + + // if the list is non-empty and the head is valid + if (curr != invalid_index && m_map.valid_at(curr) ) { + size_type prev = curr; + curr = m_map.m_next_index[prev]; + + while (curr != invalid_index) { + next = m_map.m_next_index[curr]; + if (m_map.valid_at(curr)) { + prev = curr; + } + else { + // remove curr from list + m_map.m_next_index[prev] = next; + m_map.m_next_index[curr] = invalid_index; + m_map.m_keys[curr] = key_type(); + if (map_type::is_set) m_map.m_values[curr] = value_type(); + } + curr = next; + } + } + } +}; + +template +struct UnorderedMapHistogram +{ + typedef UMap map_type; + typedef typename map_type::device_type device_type; + typedef typename map_type::size_type size_type; + + typedef View histogram_view; + typedef typename histogram_view::HostMirror host_histogram_view; + + map_type m_map; + histogram_view m_length; + histogram_view m_distance; + histogram_view m_block_distance; + + UnorderedMapHistogram( map_type const& map) + : m_map(map) + , m_length("UnorderedMap Histogram") + , m_distance("UnorderedMap Histogram") + , m_block_distance("UnorderedMap Histogram") + {} + + void calculate() + { + parallel_for(m_map.m_hash_lists.size(), *this); + } + + void clear() + { + Kokkos::deep_copy(m_length, 0); + Kokkos::deep_copy(m_distance, 0); + Kokkos::deep_copy(m_block_distance, 0); + } + + void print_length(std::ostream &out) + { + host_histogram_view host_copy = create_mirror_view(m_length); + Kokkos::deep_copy(host_copy, m_length); + + for (int i=0, size = host_copy.size(); i +struct UnorderedMapPrint +{ + typedef UMap map_type; + typedef typename map_type::device_type device_type; + typedef typename map_type::size_type size_type; + + map_type m_map; + + UnorderedMapPrint( map_type const& map) + : m_map(map) + {} + + void apply() + { + parallel_for(m_map.m_hash_lists.size(), *this); + } + + KOKKOS_INLINE_FUNCTION + void operator()( size_type i ) const + { + const size_type invalid_index = map_type::invalid_index; + + uint32_t list = m_map.m_hash_lists(i); + for (size_type curr = list, ii=0; curr != invalid_index; curr = m_map.m_next_index[curr], ++ii) { + printf("%d[%d]: %d->%d\n", list, ii, m_map.key_at(curr), m_map.value_at(curr)); + } + } +}; + +template +struct UnorderedMapCanAssign : public false_ {}; + +template +struct UnorderedMapCanAssign : public true_ {}; + +template +struct UnorderedMapCanAssign : public true_ {}; + +template +struct UnorderedMapCanAssign : public true_ {}; + +template +struct UnorderedMapCanAssign : public true_ {}; + + +}} //Kokkos::Impl + +#endif // KOKKOS_UNORDERED_MAP_IMPL_HPP diff --git a/lib/kokkos/core/src/Cuda/Kokkos_CudaExec.hpp b/lib/kokkos/core/src/Cuda/Kokkos_CudaExec.hpp new file mode 100755 index 0000000000..027155bc56 --- /dev/null +++ b/lib/kokkos/core/src/Cuda/Kokkos_CudaExec.hpp @@ -0,0 +1,237 @@ +/* +//@HEADER +// ************************************************************************ +// +// Kokkos +// Manycore Performance-Portable Multidimensional Arrays +// +// Copyright (2012) Sandia Corporation +// +// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +// the U.S. Government retains certain rights in this software. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// 1. Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// 3. Neither the name of the Corporation nor the names of the +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Questions? Contact H. Carter Edwards (hcedwar@sandia.gov) +// +// ************************************************************************ +//@HEADER +*/ + +#ifndef KOKKOS_CUDAEXEC_HPP +#define KOKKOS_CUDAEXEC_HPP + +#include +#include +#include +#include + +//---------------------------------------------------------------------------- +//---------------------------------------------------------------------------- + +namespace Kokkos { +namespace Impl { + +struct CudaTraits { + enum { WarpSize = 32 /* 0x0020 */ }; + enum { WarpIndexMask = 0x001f /* Mask for warpindex */ }; + enum { WarpIndexShift = 5 /* WarpSize == 1 << WarpShift */ }; + + enum { SharedMemoryBanks = 32 /* Compute device 2.0 */ }; + enum { SharedMemoryCapacity = 0x0C000 /* 48k shared / 16k L1 Cache */ }; + enum { SharedMemoryUsage = 0x04000 /* 16k shared / 48k L1 Cache */ }; + + enum { UpperBoundGridCount = 65535 /* Hard upper bound */ }; + enum { ConstantMemoryCapacity = 0x010000 /* 64k bytes */ }; + enum { ConstantMemoryUsage = 0x008000 /* 32k bytes */ }; + enum { ConstantMemoryCache = 0x002000 /* 8k bytes */ }; + + typedef unsigned long + ConstantGlobalBufferType[ ConstantMemoryUsage / sizeof(unsigned long) ]; + + enum { ConstantMemoryUseThreshold = 0x000200 /* 512 bytes */ }; + + KOKKOS_INLINE_FUNCTION static + CudaSpace::size_type warp_count( CudaSpace::size_type i ) + { return ( i + WarpIndexMask ) >> WarpIndexShift ; } + + KOKKOS_INLINE_FUNCTION static + CudaSpace::size_type warp_align( CudaSpace::size_type i ) + { + enum { Mask = ~CudaSpace::size_type( WarpIndexMask ) }; + return ( i + WarpIndexMask ) & Mask ; + } +}; + +//---------------------------------------------------------------------------- + +CudaSpace::size_type cuda_internal_maximum_warp_count(); +CudaSpace::size_type cuda_internal_maximum_grid_count(); +CudaSpace::size_type cuda_internal_maximum_shared_words(); + +CudaSpace::size_type * cuda_internal_scratch_flags( const CudaSpace::size_type size ); +CudaSpace::size_type * cuda_internal_scratch_space( const CudaSpace::size_type size ); +CudaSpace::size_type * cuda_internal_scratch_unified( const CudaSpace::size_type size ); + +} // namespace Impl +} // namespace Kokkos + +//---------------------------------------------------------------------------- +//---------------------------------------------------------------------------- + +#if defined( __CUDACC__ ) + +/** \brief Access to constant memory on the device */ +#ifdef KOKKOS_CUDA_USE_RELOCATABLE_DEVICE_CODE +extern +#endif +__device__ __constant__ +Kokkos::Impl::CudaTraits::ConstantGlobalBufferType +kokkos_impl_cuda_constant_memory_buffer ; + +template< typename T > +inline +__device__ +T * kokkos_impl_cuda_shared_memory() +{ extern __shared__ Kokkos::CudaSpace::size_type sh[]; return (T*) sh ; } + +namespace Kokkos { +namespace Impl { + +//---------------------------------------------------------------------------- +// See section B.17 of Cuda C Programming Guide Version 3.2 +// for discussion of +// __launch_bounds__(maxThreadsPerBlock,minBlocksPerMultiprocessor) +// function qualifier which could be used to improve performance. +//---------------------------------------------------------------------------- +// Maximize L1 cache and minimize shared memory: +// cudaFuncSetCacheConfig(MyKernel, cudaFuncCachePreferL1 ); +// For 2.0 capability: 48 KB L1 and 16 KB shared +//---------------------------------------------------------------------------- + +template< class DriverType > +__global__ +static void cuda_parallel_launch_constant_memory() +{ + const DriverType & driver = + *((const DriverType *) kokkos_impl_cuda_constant_memory_buffer ); + + driver(); +} + +template< class DriverType > +__global__ +static void cuda_parallel_launch_local_memory( const DriverType driver ) +{ + driver(); +} + +template < class DriverType , + bool Large = ( CudaTraits::ConstantMemoryUseThreshold < sizeof(DriverType) ) > +struct CudaParallelLaunch ; + +template < class DriverType > +struct CudaParallelLaunch< DriverType , true > { + + inline + CudaParallelLaunch( const DriverType & driver + , const dim3 & grid + , const dim3 & block + , const int shmem + , const cudaStream_t stream = 0 ) + { + if ( grid.x && ( block.x * block.y * block.z ) ) { + + if ( sizeof( Kokkos::Impl::CudaTraits::ConstantGlobalBufferType ) < + sizeof( DriverType ) ) { + Kokkos::Impl::throw_runtime_exception( std::string("CudaParallelLaunch FAILED: Functor is too large") ); + } + + if ( CudaTraits::SharedMemoryCapacity < shmem ) { + Kokkos::Impl::throw_runtime_exception( std::string("CudaParallelLaunch FAILED: shared memory request is too large") ); + } + else if ( shmem ) { + cudaFuncSetCacheConfig( cuda_parallel_launch_constant_memory< DriverType > , cudaFuncCachePreferShared ); + } else { + cudaFuncSetCacheConfig( cuda_parallel_launch_constant_memory< DriverType > , cudaFuncCachePreferL1 ); + } + + // Copy functor to constant memory on the device + cudaMemcpyToSymbol( kokkos_impl_cuda_constant_memory_buffer , & driver , sizeof(DriverType) ); + + // Invoke the driver function on the device + cuda_parallel_launch_constant_memory< DriverType ><<< grid , block , shmem , stream >>>(); + +#if defined( KOKKOS_EXPRESSION_CHECK ) + Kokkos::Cuda::fence(); +#endif + } + } +}; + +template < class DriverType > +struct CudaParallelLaunch< DriverType , false > { + + inline + CudaParallelLaunch( const DriverType & driver + , const dim3 & grid + , const dim3 & block + , const int shmem + , const cudaStream_t stream = 0 ) + { + if ( grid.x && ( block.x * block.y * block.z ) ) { + + if ( CudaTraits::SharedMemoryCapacity < shmem ) { + Kokkos::Impl::throw_runtime_exception( std::string("CudaParallelLaunch FAILED: shared memory request is too large") ); + } + else if ( shmem ) { + cudaFuncSetCacheConfig( cuda_parallel_launch_constant_memory< DriverType > , cudaFuncCachePreferShared ); + } else { + cudaFuncSetCacheConfig( cuda_parallel_launch_constant_memory< DriverType > , cudaFuncCachePreferL1 ); + } + + cuda_parallel_launch_local_memory< DriverType ><<< grid , block , shmem , stream >>>( driver ); + +#if defined( KOKKOS_EXPRESSION_CHECK ) + Kokkos::Cuda::fence(); +#endif + } + } +}; + +//---------------------------------------------------------------------------- + +} // namespace Impl +} // namespace Kokkos + +//---------------------------------------------------------------------------- +//---------------------------------------------------------------------------- + +#endif /* defined( __CUDACC__ ) */ + +#endif /* #ifndef KOKKOS_CUDAEXEC_HPP */ diff --git a/lib/kokkos/core/src/Cuda/Kokkos_CudaSpace.cpp b/lib/kokkos/core/src/Cuda/Kokkos_CudaSpace.cpp new file mode 100755 index 0000000000..46fbf10830 --- /dev/null +++ b/lib/kokkos/core/src/Cuda/Kokkos_CudaSpace.cpp @@ -0,0 +1,591 @@ +/* +//@HEADER +// ************************************************************************ +// +// Kokkos: Manycore Performance-Portable Multidimensional Arrays +// Copyright (2012) Sandia Corporation +// +// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +// the U.S. Government retains certain rights in this software. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// 1. Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// 3. Neither the name of the Corporation nor the names of the +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Questions? Contact H. Carter Edwards (hcedwar@sandia.gov) +// +// ************************************************************************ +//@HEADER +*/ + +#include +#include +#include +#include +#include + +/* only compile this file if CUDA is enabled for Kokkos */ +#ifdef KOKKOS_HAVE_CUDA + +#include +#include + +#include +#include +#include + +/*--------------------------------------------------------------------------*/ +/*--------------------------------------------------------------------------*/ + +namespace Kokkos { +namespace Impl { + +DeepCopy::DeepCopy( void * dst , const void * src , size_t n ) +{ CUDA_SAFE_CALL( cudaMemcpy( dst , src , n , cudaMemcpyDefault ) ); } + +DeepCopy::DeepCopy( const Cuda & instance , void * dst , const void * src , size_t n ) +{ CUDA_SAFE_CALL( cudaMemcpyAsync( dst , src , n , cudaMemcpyDefault , instance.m_stream ) ); } + +DeepCopy::DeepCopy( void * dst , const void * src , size_t n ) +{ CUDA_SAFE_CALL( cudaMemcpy( dst , src , n , cudaMemcpyDefault ) ); } + +DeepCopy::DeepCopy( const Cuda & instance , void * dst , const void * src , size_t n ) +{ CUDA_SAFE_CALL( cudaMemcpyAsync( dst , src , n , cudaMemcpyDefault , instance.m_stream ) ); } + +DeepCopy::DeepCopy( void * dst , const void * src , size_t n ) +{ CUDA_SAFE_CALL( cudaMemcpy( dst , src , n , cudaMemcpyDefault ) ); } + +DeepCopy::DeepCopy( const Cuda & instance , void * dst , const void * src , size_t n ) +{ CUDA_SAFE_CALL( cudaMemcpyAsync( dst , src , n , cudaMemcpyDefault , instance.m_stream ) ); } + +} // namespace Impl +} // namespace Kokkos + +/*--------------------------------------------------------------------------*/ +/*--------------------------------------------------------------------------*/ + +namespace Kokkos { +namespace Impl { +namespace { + +class CudaMemoryTracking { +public: + + enum SpaceTag { CudaSpaceTag , CudaUVMSpaceTag , CudaHostPinnedSpaceTag }; + + struct Attribute { + + Kokkos::Impl::cuda_texture_object_type m_tex_obj ; + int m_tex_flag ; + + Attribute() : m_tex_obj(0), m_tex_flag(0) {} + + ~Attribute() + { + if ( m_tex_flag ) { + cudaDestroyTextureObject( m_tex_obj ); + m_tex_obj = 0 ; + m_tex_flag = 0 ; + } + } + + cudaError create( void * const arg_alloc_ptr + , size_t const arg_byte_size + , cudaChannelFormatDesc const & arg_desc + ) + { + cudaError cuda_status = cudaSuccess ; + + if ( 0 == m_tex_flag ) { + + cuda_status = cudaDeviceSynchronize(); + + if ( cudaSuccess == cuda_status ) { + struct cudaResourceDesc resDesc ; + struct cudaTextureDesc texDesc ; + + memset( & resDesc , 0 , sizeof(resDesc) ); + memset( & texDesc , 0 , sizeof(texDesc) ); + + resDesc.resType = cudaResourceTypeLinear ; + resDesc.res.linear.desc = arg_desc ; + resDesc.res.linear.sizeInBytes = arg_byte_size ; + resDesc.res.linear.devPtr = arg_alloc_ptr ; + + cuda_status = cudaCreateTextureObject( & m_tex_obj , & resDesc, & texDesc, NULL); + } + + if ( cudaSuccess == cuda_status ) { cuda_status = cudaDeviceSynchronize(); } + + if ( cudaSuccess == cuda_status ) { m_tex_flag = 1 ; } + } + + return cuda_status ; + } + }; + + typedef Kokkos::Impl::MemoryTracking< Attribute > tracking_type ; + typedef typename Kokkos::Impl::MemoryTracking< Attribute >::Entry entry_type ; + + bool available() const + { +#if defined( CUDA_VERSION ) && ( 6000 <= CUDA_VERSION ) + enum { UVM_available = true }; +#else + enum { UVM_available = false }; +#endif + + return ( m_space_tag != CudaUVMSpaceTag ) || UVM_available ; + } + +private: + + tracking_type m_tracking ; + SpaceTag const m_space_tag ; + + + cudaError cuda_malloc( void ** ptr , size_t byte_size ) const + { + cudaError result = cudaSuccess ; + + switch( m_space_tag ) { + case CudaSpaceTag : + result = cudaMalloc( ptr , byte_size ); + break ; + case CudaUVMSpaceTag : +#if defined( CUDA_VERSION ) && ( 6000 <= CUDA_VERSION ) + result = cudaMallocManaged( ptr, byte_size, cudaMemAttachGlobal ); +#else + Kokkos::Impl::throw_runtime_exception( std::string("CUDA VERSION does not support UVM") ); +#endif + break ; + case CudaHostPinnedSpaceTag : + result = cudaHostAlloc( ptr , byte_size , cudaHostAllocDefault ); + break ; + } + + return result ; + } + + cudaError cuda_free( void * ptr ) const + { + cudaError result = cudaSuccess ; + + switch( m_space_tag ) { + case CudaSpaceTag : + case CudaUVMSpaceTag : + result = cudaFree( ptr ); + break ; + case CudaHostPinnedSpaceTag : + result = cudaFreeHost( ptr ); + break ; + } + return result ; + } + +public : + + CudaMemoryTracking( const SpaceTag arg_tag , const char * const arg_label ) + : m_tracking( arg_label ) + , m_space_tag( arg_tag ) + {} + + void print( std::ostream & oss , const std::string & lead ) const + { m_tracking.print( oss , lead ); } + + const char * query_label( const void * ptr ) const + { + static const char error[] = "" ; + entry_type * const entry = m_tracking.query( ptr ); + return entry ? entry->label() : error ; + } + + int count(const void * ptr) const { + entry_type * const entry = m_tracking.query( ptr ); + return entry ? entry->count() : 0 ; + } + + void * allocate( const std::string & label , const size_t byte_size ) + { + void * ptr = 0 ; + + if ( byte_size ) { + + const bool ok_parallel = ! HostSpace::in_parallel(); + + cudaError cuda_status = cudaSuccess ; + + if ( ok_parallel ) { + + cuda_status = cudaDeviceSynchronize(); + + if ( cudaSuccess == cuda_status ) { cuda_status = CudaMemoryTracking::cuda_malloc( & ptr , byte_size ); } + if ( cudaSuccess == cuda_status ) { cuda_status = cudaDeviceSynchronize(); } + } + + if ( ok_parallel && ( cudaSuccess == cuda_status ) ) { + m_tracking.insert( label , ptr , byte_size ); + } + else { + std::ostringstream msg ; + msg << m_tracking.label() + << "::allocate( " + << label + << " , " << byte_size + << " ) FAILURE : " ; + if ( ! ok_parallel ) { + msg << "called within a parallel functor" ; + } + else { + msg << " CUDA ERROR \"" << cudaGetErrorString(cuda_status) << "\"" ; + } + Kokkos::Impl::throw_runtime_exception( msg.str() ); + } + } + + return ptr ; + } + + void decrement( const void * ptr ) + { + const bool ok_parallel = ! HostSpace::in_parallel(); + + cudaError cuda_status = cudaSuccess ; + + if ( ok_parallel ) { + + cuda_status = cudaDeviceSynchronize(); + + void * const alloc_ptr = ( cudaSuccess == cuda_status ) ? m_tracking.decrement( ptr ) : (void *) 0 ; + + if ( alloc_ptr ) { + if ( cudaSuccess == cuda_status ) { cuda_status = CudaMemoryTracking::cuda_free( alloc_ptr ); } + if ( cudaSuccess == cuda_status ) { cuda_status = cudaDeviceSynchronize(); } + } + } + + if ( ( ! ok_parallel ) || ( cudaSuccess != cuda_status ) ) { + std::ostringstream msg ; + msg << m_tracking.label() << "::decrement( " << ptr << " ) FAILURE : " ; + if ( ! ok_parallel ) { + msg << "called within a parallel functor" ; + } + else { + msg << " CUDA ERROR \"" << cudaGetErrorString(cuda_status) << "\"" ; + } + std::cerr << msg.str() << std::endl ; + } + } + + void increment( const void * ptr ) + { + const bool ok_parallel = ! HostSpace::in_parallel(); + + if ( ok_parallel ) { + m_tracking.increment( ptr ); + } + else { + std::ostringstream msg ; + msg << m_tracking.label() << "::increment(" << ptr + << ") FAILURE :called within a parallel functor" ; + Kokkos::Impl::throw_runtime_exception( msg.str() ); + } + } + + + inline + void texture_object_attach( const void * const arg_ptr + , const unsigned arg_type_size + , const cudaChannelFormatDesc & arg_desc + , ::cudaTextureObject_t * const arg_tex_obj + , void const ** const arg_alloc_ptr + , int * const arg_offset + ) + { + static const size_t max_array_len = 1 << 28 ; + + *arg_tex_obj = 0 ; + *arg_alloc_ptr = 0 ; + *arg_offset = 0 ; + + if ( arg_ptr ) { + + // Can only create texture object on device architure 3.0 or better + const bool ok_dev_arch = 300 <= Cuda::device_arch(); + const bool ok_parallel = ok_dev_arch && ! HostSpace::in_parallel(); + + entry_type * const entry = ok_parallel ? m_tracking.query( arg_ptr ) : (entry_type *) 0 ; + + const size_t offset = entry ? ( reinterpret_cast(arg_ptr) - + reinterpret_cast(entry->m_alloc_ptr) ) : 0 ; + + const bool ok_offset = entry && ( 0 == ( offset % arg_type_size ) ); + const bool ok_count = ok_offset && ( entry->m_alloc_size / arg_type_size < max_array_len ); + + cudaError cuda_status = cudaSuccess ; + + if ( ok_count ) { + cuda_status = entry->m_attribute.create( entry->m_alloc_ptr , entry->m_alloc_size , arg_desc ); + } + + if ( cudaSuccess == cuda_status ) { + *arg_tex_obj = entry->m_attribute.m_tex_obj ; + *arg_alloc_ptr = entry->m_alloc_ptr ; + *arg_offset = offset / arg_type_size ; + } + else { + std::ostringstream msg ; + msg << m_tracking.label() + << "::texture_object_attach(" << arg_ptr << ") FAILED :" ; + if ( ! ok_dev_arch ) { + msg << " cuda architecture " << Cuda::device_arch() + << " does not support texture objects" ; + } + else if ( ! ok_parallel ) { + msg << " called within a parallel functor" ; + } + else if ( 0 == entry ) { + msg << " pointer not tracked" ; + } + else if ( ! ok_offset ) { + msg << " pointer not properly aligned" ; + } + else if ( ! ok_count ) { + msg << " array too large for texture object" ; + } + else { + msg << " CUDA ERROR \"" << cudaGetErrorString(cuda_status) << "\"" ; + } + Kokkos::Impl::throw_runtime_exception( msg.str() ); + } + } + } +}; + +//---------------------------------------------------------------------------- + +CudaMemoryTracking & +cuda_space_singleton() +{ + static CudaMemoryTracking s( CudaMemoryTracking::CudaSpaceTag , "Kokkos::CudaSpace"); + return s ; +} + +CudaMemoryTracking & +cuda_uvm_space_singleton() +{ + static CudaMemoryTracking s( CudaMemoryTracking::CudaUVMSpaceTag , "Kokkos::CudaUVMSpace"); + return s ; +} + +CudaMemoryTracking & +cuda_host_pinned_space_singleton() +{ + static CudaMemoryTracking s( CudaMemoryTracking::CudaHostPinnedSpaceTag , "Kokkos::CudaHostPinnedSpace"); + return s ; +} + +} +} // namespace Impl +} // namespace Kokkos + +/*--------------------------------------------------------------------------*/ +/*--------------------------------------------------------------------------*/ + +namespace Kokkos { + +void * CudaSpace::allocate( const std::string & label , const size_t size ) +{ + return Impl::cuda_space_singleton().allocate( label , size ); +} + +void CudaSpace::decrement( const void * ptr ) +{ + Impl::cuda_space_singleton().decrement( ptr ); +} + + +void CudaSpace::increment( const void * ptr ) +{ + Impl::cuda_space_singleton().increment( ptr ); +} + +void CudaSpace::print_memory_view( std::ostream & oss ) +{ + Impl::cuda_space_singleton().print( oss , std::string(" ") ); +} + +int CudaSpace::count( const void * ptr ) { + if ( ! HostSpace::in_parallel() ) { + return Impl::cuda_space_singleton().count(ptr); + } + else { + Kokkos::Impl::throw_runtime_exception( std::string("Kokkos::CudaSpace::count called within a parallel functor") ); + return -1; + } +} + +std::string CudaSpace::query_label( const void * p ) +{ + return std::string( Impl::cuda_space_singleton().query_label(p) ); +} + +void CudaSpace::texture_object_attach( const void * const arg_ptr + , const unsigned arg_type_size + , ::cudaChannelFormatDesc const & arg_desc + , ::cudaTextureObject_t * const arg_tex_obj + , void const ** const arg_alloc_ptr + , int * const arg_offset + ) +{ + Impl::cuda_space_singleton().texture_object_attach( arg_ptr , arg_type_size , arg_desc , arg_tex_obj , arg_alloc_ptr , arg_offset ); +} + +void CudaSpace::access_error() +{ + const std::string msg("Kokkos::CudaSpace::access_error attempt to execute Cuda function from non-Cuda space" ); + + Kokkos::Impl::throw_runtime_exception( msg ); +} + +void CudaSpace::access_error( const void * const ptr ) +{ + std::ostringstream msg ; + msg << "Kokkos::CudaSpace::access_error:" ; + msg << " attempt to access Cuda-data labeled(" ; + msg << query_label( ptr ) ; + msg << ") from non-Cuda execution" ; + Kokkos::Impl::throw_runtime_exception( msg.str() ); +} + +} // namespace Kokkos + +/*--------------------------------------------------------------------------*/ +/*--------------------------------------------------------------------------*/ + +namespace Kokkos { + +bool CudaUVMSpace::available() +{ + return Impl::cuda_uvm_space_singleton().available(); +} + +void * CudaUVMSpace::allocate( const std::string & label , const size_t size ) +{ + return Impl::cuda_uvm_space_singleton().allocate( label , size ); +} + +void CudaUVMSpace::decrement( const void * ptr ) +{ + Impl::cuda_uvm_space_singleton().decrement( ptr ); +} + + +void CudaUVMSpace::increment( const void * ptr ) +{ + Impl::cuda_uvm_space_singleton().increment( ptr ); +} + +int CudaUVMSpace::count( const void * ptr ) { + if ( ! HostSpace::in_parallel() ) { + return Impl::cuda_uvm_space_singleton().count(ptr); + } + else { + Kokkos::Impl::throw_runtime_exception( std::string("Kokkos::CudaUVMSpace::count called within a parallel functor") ); + return -1; + } +} + +void CudaUVMSpace::print_memory_view( std::ostream & oss ) +{ + Impl::cuda_uvm_space_singleton().print( oss , std::string(" ") ); +} + +std::string CudaUVMSpace::query_label( const void * p ) +{ + return std::string( Impl::cuda_uvm_space_singleton().query_label(p) ); +} + +void CudaUVMSpace::texture_object_attach( const void * const arg_ptr + , const unsigned arg_type_size + , ::cudaChannelFormatDesc const & arg_desc + , ::cudaTextureObject_t * const arg_tex_obj + , void const ** const arg_alloc_ptr + , int * const arg_offset + ) +{ + Impl::cuda_uvm_space_singleton().texture_object_attach( arg_ptr , arg_type_size , arg_desc , arg_tex_obj , arg_alloc_ptr , arg_offset ); +} + +} // namespace Kokkos + +/*--------------------------------------------------------------------------*/ +/*--------------------------------------------------------------------------*/ + +namespace Kokkos { + +void * CudaHostPinnedSpace::allocate( const std::string & label , const size_t size ) +{ + return Impl::cuda_host_pinned_space_singleton().allocate( label , size ); +} + +void CudaHostPinnedSpace::decrement( const void * ptr ) +{ + Impl::cuda_host_pinned_space_singleton().decrement( ptr ); +} + + +void CudaHostPinnedSpace::increment( const void * ptr ) +{ + Impl::cuda_host_pinned_space_singleton().increment( ptr ); +} + +int CudaHostPinnedSpace::count( const void * ptr ) { + if ( ! HostSpace::in_parallel() ) { + return Impl::cuda_uvm_space_singleton().count(ptr); + } + else { + Kokkos::Impl::throw_runtime_exception( std::string("Kokkos::CudaHostPinnedSpace::count called within a parallel functor") ); + return -1; + } +} + +void CudaHostPinnedSpace::print_memory_view( std::ostream & oss ) +{ + Impl::cuda_host_pinned_space_singleton().print( oss , std::string(" ") ); +} + +std::string CudaHostPinnedSpace::query_label( const void * p ) +{ + return std::string( Impl::cuda_host_pinned_space_singleton().query_label(p) ); +} + +} // namespace Kokkos + +#endif // KOKKOS_HAVE_CUDA +/*--------------------------------------------------------------------------*/ +/*--------------------------------------------------------------------------*/ + diff --git a/lib/kokkos/core/src/Cuda/Kokkos_Cuda_Impl.cpp b/lib/kokkos/core/src/Cuda/Kokkos_Cuda_Impl.cpp new file mode 100755 index 0000000000..87a2e95ed5 --- /dev/null +++ b/lib/kokkos/core/src/Cuda/Kokkos_Cuda_Impl.cpp @@ -0,0 +1,670 @@ +/* +//@HEADER +// ************************************************************************ +// +// Kokkos: Manycore Performance-Portable Multidimensional Arrays +// Copyright (2012) Sandia Corporation +// +// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +// the U.S. Government retains certain rights in this software. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// 1. Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// 3. Neither the name of the Corporation nor the names of the +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Questions? Contact H. Carter Edwards (hcedwar@sandia.gov) +// +// ************************************************************************ +//@HEADER +*/ + +/*--------------------------------------------------------------------------*/ +/* Kokkos interfaces */ + +#include + +/* only compile this file if CUDA is enabled for Kokkos */ +#ifdef KOKKOS_HAVE_CUDA + +#include +#include + +/*--------------------------------------------------------------------------*/ +/* Standard 'C' libraries */ +#include + +/* Standard 'C++' libraries */ +#include +#include +#include +#include + +#ifdef KOKKOS_CUDA_USE_RELOCATABLE_DEVICE_CODE +__device__ __constant__ +Kokkos::Impl::CudaTraits::ConstantGlobalBufferType +kokkos_impl_cuda_constant_memory_buffer ; +#endif + +/*--------------------------------------------------------------------------*/ + +namespace Kokkos { +namespace Impl { + +namespace { + +__global__ +void query_cuda_kernel_arch( int * d_arch ) +{ +#if defined( __CUDA_ARCH__ ) + *d_arch = __CUDA_ARCH__ ; +#else + *d_arch = 0 ; +#endif +} + +/** Query what compute capability is actually launched to the device: */ +int cuda_kernel_arch() +{ + int * d_arch = 0 ; + cudaMalloc( (void **) & d_arch , sizeof(int) ); + query_cuda_kernel_arch<<<1,1>>>( d_arch ); + int arch = 0 ; + cudaMemcpy( & arch , d_arch , sizeof(int) , cudaMemcpyDefault ); + cudaFree( d_arch ); + return arch ; +} + +bool cuda_launch_blocking() +{ + const char * env = getenv("CUDA_LAUNCH_BLOCKING"); + + if (env == 0) return false; + + return atoi(env); +} + +} + +void cuda_device_synchronize() +{ + static const bool launch_blocking = cuda_launch_blocking(); + + if (!launch_blocking) { + CUDA_SAFE_CALL( cudaDeviceSynchronize() ); + } +} + +void cuda_internal_error_throw( cudaError e , const char * name, const char * file, const int line ) +{ + std::ostringstream out ; + out << name << " error: " << cudaGetErrorString(e); + if (file) { + out << " " << file << ":" << line; + } + throw_runtime_exception( out.str() ); +} + +//---------------------------------------------------------------------------- +// Some significant cuda device properties: +// +// cudaDeviceProp::name : Text label for device +// cudaDeviceProp::major : Device major number +// cudaDeviceProp::minor : Device minor number +// cudaDeviceProp::warpSize : number of threads per warp +// cudaDeviceProp::multiProcessorCount : number of multiprocessors +// cudaDeviceProp::sharedMemPerBlock : capacity of shared memory per block +// cudaDeviceProp::totalConstMem : capacity of constant memory +// cudaDeviceProp::totalGlobalMem : capacity of global memory +// cudaDeviceProp::maxGridSize[3] : maximum grid size + +// +// Section 4.4.2.4 of the CUDA Toolkit Reference Manual +// +// struct cudaDeviceProp { +// char name[256]; +// size_t totalGlobalMem; +// size_t sharedMemPerBlock; +// int regsPerBlock; +// int warpSize; +// size_t memPitch; +// int maxThreadsPerBlock; +// int maxThreadsDim[3]; +// int maxGridSize[3]; +// size_t totalConstMem; +// int major; +// int minor; +// int clockRate; +// size_t textureAlignment; +// int deviceOverlap; +// int multiProcessorCount; +// int kernelExecTimeoutEnabled; +// int integrated; +// int canMapHostMemory; +// int computeMode; +// int concurrentKernels; +// int ECCEnabled; +// int pciBusID; +// int pciDeviceID; +// int tccDriver; +// int asyncEngineCount; +// int unifiedAddressing; +// int memoryClockRate; +// int memoryBusWidth; +// int l2CacheSize; +// int maxThreadsPerMultiProcessor; +// }; + + +namespace { + + + +class CudaInternalDevices { +public: + enum { MAXIMUM_DEVICE_COUNT = 8 }; + struct cudaDeviceProp m_cudaProp[ MAXIMUM_DEVICE_COUNT ] ; + int m_cudaDevCount ; + + CudaInternalDevices(); + + static const CudaInternalDevices & singleton(); +}; + +CudaInternalDevices::CudaInternalDevices() +{ + // See 'cudaSetDeviceFlags' for host-device thread interaction + // Section 4.4.2.6 of the CUDA Toolkit Reference Manual + + CUDA_SAFE_CALL (cudaGetDeviceCount( & m_cudaDevCount ) ); + + for ( int i = 0 ; i < m_cudaDevCount ; ++i ) { + CUDA_SAFE_CALL( cudaGetDeviceProperties( m_cudaProp + i , i ) ); + } +} + +const CudaInternalDevices & CudaInternalDevices::singleton() +{ + static CudaInternalDevices self ; return self ; +} + +} + +//---------------------------------------------------------------------------- + +class CudaInternal { +private: + + CudaInternal( const CudaInternal & ); + CudaInternal & operator = ( const CudaInternal & ); + +public: + + typedef Cuda::size_type size_type ; + + int m_cudaDev ; + int m_cudaArch ; + unsigned m_maxWarpCount ; + unsigned m_maxBlock ; + unsigned m_maxSharedWords ; + size_type m_scratchSpaceCount ; + size_type m_scratchFlagsCount ; + size_type m_scratchUnifiedCount ; + size_type m_scratchUnifiedSupported ; + size_type m_streamCount ; + size_type * m_scratchSpace ; + size_type * m_scratchFlags ; + size_type * m_scratchUnified ; + cudaStream_t * m_stream ; + + + static CudaInternal & singleton(); + + int verify_is_initialized( const char * const label ) const ; + + int is_initialized() const + { return 0 != m_scratchSpace && 0 != m_scratchFlags ; } + + void initialize( int cuda_device_id , int stream_count ); + void finalize(); + + void print_configuration( std::ostream & ) const ; + + ~CudaInternal(); + + CudaInternal() + : m_cudaDev( -1 ) + , m_cudaArch( -1 ) + , m_maxWarpCount( 0 ) + , m_maxBlock( 0 ) + , m_maxSharedWords( 0 ) + , m_scratchSpaceCount( 0 ) + , m_scratchFlagsCount( 0 ) + , m_scratchUnifiedCount( 0 ) + , m_scratchUnifiedSupported( 0 ) + , m_streamCount( 0 ) + , m_scratchSpace( 0 ) + , m_scratchFlags( 0 ) + , m_scratchUnified( 0 ) + , m_stream( 0 ) + {} + + size_type * scratch_space( const size_type size ); + size_type * scratch_flags( const size_type size ); + size_type * scratch_unified( const size_type size ); +}; + +//---------------------------------------------------------------------------- + + +void CudaInternal::print_configuration( std::ostream & s ) const +{ + const CudaInternalDevices & dev_info = CudaInternalDevices::singleton(); + +#if defined( KOKKOS_HAVE_CUDA ) + s << "macro KOKKOS_HAVE_CUDA : defined" << std::endl ; +#endif +#if defined( CUDA_VERSION ) + s << "macro CUDA_VERSION = " << CUDA_VERSION + << " = version " << CUDA_VERSION / 1000 + << "." << ( CUDA_VERSION % 1000 ) / 10 + << std::endl ; +#endif + + for ( int i = 0 ; i < dev_info.m_cudaDevCount ; ++i ) { + s << "Kokkos::Cuda[ " << i << " ] " + << dev_info.m_cudaProp[i].name + << " capability " << dev_info.m_cudaProp[i].major << "." << dev_info.m_cudaProp[i].minor + << ", Total Global Memory: " << human_memory_size(dev_info.m_cudaProp[i].totalGlobalMem) + << ", Shared Memory per Block: " << human_memory_size(dev_info.m_cudaProp[i].sharedMemPerBlock); + if ( m_cudaDev == i ) s << " : Selected" ; + s << std::endl ; + } +} + +//---------------------------------------------------------------------------- + +CudaInternal::~CudaInternal() +{ + if ( m_stream || + m_scratchSpace || + m_scratchFlags || + m_scratchUnified ) { + std::cerr << "Kokkos::Cuda ERROR: Failed to call Kokkos::Cuda::finalize()" + << std::endl ; + std::cerr.flush(); + } + + m_cudaDev = -1 ; + m_cudaArch = -1 ; + m_maxWarpCount = 0 ; + m_maxBlock = 0 ; + m_maxSharedWords = 0 ; + m_scratchSpaceCount = 0 ; + m_scratchFlagsCount = 0 ; + m_scratchUnifiedCount = 0 ; + m_scratchUnifiedSupported = 0 ; + m_streamCount = 0 ; + m_scratchSpace = 0 ; + m_scratchFlags = 0 ; + m_scratchUnified = 0 ; + m_stream = 0 ; +} + +int CudaInternal::verify_is_initialized( const char * const label ) const +{ + if ( m_cudaDev < 0 ) { + std::cerr << "Kokkos::Cuda::" << label << " : ERROR device not initialized" << std::endl ; + } + return 0 <= m_cudaDev ; +} + +CudaInternal & CudaInternal::singleton() +{ + static CudaInternal self ; + return self ; +} + +void CudaInternal::initialize( int cuda_device_id , int stream_count ) +{ + enum { WordSize = sizeof(size_type) }; + + if ( ! HostSpace::execution_space::is_initialized() ) { + const std::string msg("Cuda::initialize ERROR : HostSpace::execution_space is not initialized"); + throw_runtime_exception( msg ); + } + + const CudaInternalDevices & dev_info = CudaInternalDevices::singleton(); + + const bool ok_init = 0 == m_scratchSpace || 0 == m_scratchFlags ; + + const bool ok_id = 0 <= cuda_device_id && + cuda_device_id < dev_info.m_cudaDevCount ; + + // Need device capability 2.0 or better + + const bool ok_dev = ok_id && + ( 2 <= dev_info.m_cudaProp[ cuda_device_id ].major && + 0 <= dev_info.m_cudaProp[ cuda_device_id ].minor ); + + if ( ok_init && ok_dev ) { + + const struct cudaDeviceProp & cudaProp = + dev_info.m_cudaProp[ cuda_device_id ]; + + m_cudaDev = cuda_device_id ; + + CUDA_SAFE_CALL( cudaSetDevice( m_cudaDev ) ); + CUDA_SAFE_CALL( cudaDeviceReset() ); + Kokkos::Impl::cuda_device_synchronize(); + + // Query what compute capability architecture a kernel executes: + m_cudaArch = cuda_kernel_arch(); + + if ( m_cudaArch != cudaProp.major * 100 + cudaProp.minor * 10 ) { + std::cerr << "Kokkos::Cuda::initialize WARNING: running kernels compiled for compute capability " + << ( m_cudaArch / 100 ) << "." << ( ( m_cudaArch % 100 ) / 10 ) + << " on device with compute capability " + << cudaProp.major << "." << cudaProp.minor + << " , this will likely reduce potential performance." + << std::endl ; + } + + //---------------------------------- + // Maximum number of warps, + // at most one warp per thread in a warp for reduction. + + // HCE 2012-February : + // Found bug in CUDA 4.1 that sometimes a kernel launch would fail + // if the thread count == 1024 and a functor is passed to the kernel. + // Copying the kernel to constant memory and then launching with + // thread count == 1024 would work fine. + // + // HCE 2012-October : + // All compute capabilities support at least 16 warps (512 threads). + // However, we have found that 8 warps typically gives better performance. + + m_maxWarpCount = 8 ; + + // m_maxWarpCount = cudaProp.maxThreadsPerBlock / Impl::CudaTraits::WarpSize ; + + if ( Impl::CudaTraits::WarpSize < m_maxWarpCount ) { + m_maxWarpCount = Impl::CudaTraits::WarpSize ; + } + + m_maxSharedWords = cudaProp.sharedMemPerBlock / WordSize ; + + //---------------------------------- + // Maximum number of blocks: + + m_maxBlock = m_cudaArch < 300 ? 65535 : cudaProp.maxGridSize[0] ; + + //---------------------------------- + + m_scratchUnifiedSupported = cudaProp.unifiedAddressing ; + + if ( ! m_scratchUnifiedSupported ) { + std::cout << "Kokkos::Cuda device " + << cudaProp.name << " capability " + << cudaProp.major << "." << cudaProp.minor + << " does not support unified virtual address space" + << std::endl ; + } + + //---------------------------------- + // Multiblock reduction uses scratch flags for counters + // and scratch space for partial reduction values. + // Allocate some initial space. This will grow as needed. + + { + const unsigned reduce_block_count = m_maxWarpCount * Impl::CudaTraits::WarpSize ; + + (void) scratch_unified( 16 * sizeof(size_type) ); + (void) scratch_flags( reduce_block_count * 2 * sizeof(size_type) ); + (void) scratch_space( reduce_block_count * 16 * sizeof(size_type) ); + } + //---------------------------------- + + if ( stream_count ) { + m_stream = (cudaStream_t*) malloc( stream_count * sizeof(cudaStream_t) ); + m_streamCount = stream_count ; + for ( size_type i = 0 ; i < m_streamCount ; ++i ) m_stream[i] = 0 ; + } + } + else { + + std::ostringstream msg ; + msg << "Kokkos::Cuda::initialize(" << cuda_device_id << ") FAILED" ; + + if ( ! ok_init ) { + msg << " : Already initialized" ; + } + if ( ! ok_id ) { + msg << " : Device identifier out of range " + << "[0.." << dev_info.m_cudaDevCount << "]" ; + } + else if ( ! ok_dev ) { + msg << " : Device " ; + msg << dev_info.m_cudaProp[ cuda_device_id ].major ; + msg << "." ; + msg << dev_info.m_cudaProp[ cuda_device_id ].minor ; + msg << " has insufficient capability, required 2.0 or better" ; + } + Kokkos::Impl::throw_runtime_exception( msg.str() ); + } +} + +//---------------------------------------------------------------------------- + +typedef Cuda::size_type ScratchGrain[ Impl::CudaTraits::WarpSize ] ; +enum { sizeScratchGrain = sizeof(ScratchGrain) }; + + +Cuda::size_type * +CudaInternal::scratch_flags( const Cuda::size_type size ) +{ + if ( verify_is_initialized("scratch_flags") && m_scratchFlagsCount * sizeScratchGrain < size ) { + + CudaSpace::decrement( m_scratchFlags ); + + m_scratchFlagsCount = ( size + sizeScratchGrain - 1 ) / sizeScratchGrain ; + + m_scratchFlags = (size_type *) + CudaSpace::allocate( std::string("InternalScratchFlags") , sizeof( ScratchGrain ) * m_scratchFlagsCount ); + + CUDA_SAFE_CALL( cudaMemset( m_scratchFlags , 0 , m_scratchFlagsCount * sizeScratchGrain ) ); + } + + return m_scratchFlags ; +} + +Cuda::size_type * +CudaInternal::scratch_space( const Cuda::size_type size ) +{ + if ( verify_is_initialized("scratch_space") && m_scratchSpaceCount * sizeScratchGrain < size ) { + + CudaSpace::decrement( m_scratchSpace ); + + m_scratchSpaceCount = ( size + sizeScratchGrain - 1 ) / sizeScratchGrain ; + + m_scratchSpace = (size_type *) + CudaSpace::allocate( std::string("InternalScratchSpace") , sizeof( ScratchGrain ) * m_scratchSpaceCount ); + } + + return m_scratchSpace ; +} + +Cuda::size_type * +CudaInternal::scratch_unified( const Cuda::size_type size ) +{ + if ( verify_is_initialized("scratch_unified") && + m_scratchUnifiedSupported && m_scratchUnifiedCount * sizeScratchGrain < size ) { + + CudaHostPinnedSpace::decrement( m_scratchUnified ); + + m_scratchUnifiedCount = ( size + sizeScratchGrain - 1 ) / sizeScratchGrain ; + + m_scratchUnified = (size_type *) + CudaHostPinnedSpace::allocate( std::string("InternalScratchUnified") , sizeof( ScratchGrain ) * m_scratchUnifiedCount ); + } + + return m_scratchUnified ; +} + +//---------------------------------------------------------------------------- + +void CudaInternal::finalize() +{ + if ( 0 != m_scratchSpace || 0 != m_scratchFlags ) { + + if ( m_stream ) { + for ( size_type i = 1 ; i < m_streamCount ; ++i ) { + cudaStreamDestroy( m_stream[i] ); + m_stream[i] = 0 ; + } + free( m_stream ); + } + + CudaSpace::decrement( m_scratchSpace ); + CudaSpace::decrement( m_scratchFlags ); + CudaHostPinnedSpace::decrement( m_scratchUnified ); + + m_cudaDev = -1 ; + m_maxWarpCount = 0 ; + m_maxBlock = 0 ; + m_maxSharedWords = 0 ; + m_scratchSpaceCount = 0 ; + m_scratchFlagsCount = 0 ; + m_scratchUnifiedCount = 0 ; + m_streamCount = 0 ; + m_scratchSpace = 0 ; + m_scratchFlags = 0 ; + m_scratchUnified = 0 ; + m_stream = 0 ; + } +} + +//---------------------------------------------------------------------------- + +Cuda::size_type cuda_internal_maximum_warp_count() +{ return CudaInternal::singleton().m_maxWarpCount ; } + +Cuda::size_type cuda_internal_maximum_grid_count() +{ return CudaInternal::singleton().m_maxBlock ; } + +Cuda::size_type cuda_internal_maximum_shared_words() +{ return CudaInternal::singleton().m_maxSharedWords ; } + +Cuda::size_type * cuda_internal_scratch_space( const Cuda::size_type size ) +{ return CudaInternal::singleton().scratch_space( size ); } + +Cuda::size_type * cuda_internal_scratch_flags( const Cuda::size_type size ) +{ return CudaInternal::singleton().scratch_flags( size ); } + +Cuda::size_type * cuda_internal_scratch_unified( const Cuda::size_type size ) +{ return CudaInternal::singleton().scratch_unified( size ); } + + +} // namespace Impl +} // namespace Kokkos + +//---------------------------------------------------------------------------- + +namespace Kokkos { + +Cuda::size_type Cuda::detect_device_count() +{ return Impl::CudaInternalDevices::singleton().m_cudaDevCount ; } + +int Cuda::is_initialized() +{ return Impl::CudaInternal::singleton().is_initialized(); } + +void Cuda::initialize( const Cuda::SelectDevice config , size_t num_instances ) +{ Impl::CudaInternal::singleton().initialize( config.cuda_device_id , num_instances ); } + +std::vector +Cuda::detect_device_arch() +{ + const Impl::CudaInternalDevices & s = Impl::CudaInternalDevices::singleton(); + + std::vector output( s.m_cudaDevCount ); + + for ( int i = 0 ; i < s.m_cudaDevCount ; ++i ) { + output[i] = s.m_cudaProp[i].major * 100 + s.m_cudaProp[i].minor ; + } + + return output ; +} + +Cuda::size_type Cuda::device_arch() +{ + const int dev_id = Impl::CudaInternal::singleton().m_cudaDev ; + + int dev_arch = 0 ; + + if ( 0 <= dev_id ) { + const struct cudaDeviceProp & cudaProp = + Impl::CudaInternalDevices::singleton().m_cudaProp[ dev_id ] ; + + dev_arch = cudaProp.major * 100 + cudaProp.minor ; + } + + return dev_arch ; +} + +void Cuda::finalize() +{ Impl::CudaInternal::singleton().finalize(); } + +Cuda::Cuda() + : m_device( Impl::CudaInternal::singleton().m_cudaDev ) + , m_stream( 0 ) +{ + Impl::CudaInternal::singleton().verify_is_initialized( "Cuda instance constructor" ); +} + +Cuda::Cuda( const int instance_id ) + : m_device( Impl::CudaInternal::singleton().m_cudaDev ) + , m_stream( + Impl::CudaInternal::singleton().verify_is_initialized( "Cuda instance constructor" ) + ? Impl::CudaInternal::singleton().m_stream[ instance_id % Impl::CudaInternal::singleton().m_streamCount ] + : 0 ) +{} + +void Cuda::print_configuration( std::ostream & s , const bool ) +{ Impl::CudaInternal::singleton().print_configuration( s ); } + +bool Cuda::sleep() { return false ; } + +bool Cuda::wake() { return true ; } + +void Cuda::fence() +{ + Kokkos::Impl::cuda_device_synchronize(); +} + +} // namespace Kokkos + +#endif // KOKKOS_HAVE_CUDA +//---------------------------------------------------------------------------- + diff --git a/lib/kokkos/core/src/Cuda/Kokkos_Cuda_Internal.hpp b/lib/kokkos/core/src/Cuda/Kokkos_Cuda_Internal.hpp new file mode 100755 index 0000000000..284e71decd --- /dev/null +++ b/lib/kokkos/core/src/Cuda/Kokkos_Cuda_Internal.hpp @@ -0,0 +1,171 @@ +/* +//@HEADER +// ************************************************************************ +// +// Kokkos: Manycore Performance-Portable Multidimensional Arrays +// Copyright (2012) Sandia Corporation +// +// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +// the U.S. Government retains certain rights in this software. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// 1. Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// 3. Neither the name of the Corporation nor the names of the +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Questions? Contact H. Carter Edwards (hcedwar@sandia.gov) +// +// ************************************************************************ +//@HEADER +*/ + +#ifndef KOKKOS_CUDA_INTERNAL_HPP +#define KOKKOS_CUDA_INTERNAL_HPP + +namespace Kokkos { +namespace Impl { + +void cuda_internal_error_throw( cudaError e , const char * name, const char * file = NULL, const int line = 0 ); + +void cuda_device_synchronize(); + +inline +void cuda_internal_safe_call( cudaError e , const char * name, const char * file = NULL, const int line = 0) +{ + if ( cudaSuccess != e ) { cuda_internal_error_throw( e , name, file, line ); } +} + +template +int cuda_get_max_block_size(const typename DriverType::functor_type & f) { +#if ( CUDA_VERSION < 6050 ) + return 256; +#else + bool Large = ( CudaTraits::ConstantMemoryUseThreshold < sizeof(DriverType) ); + + int numBlocks; + if(Large) { + int blockSize=32; + int sharedmem = FunctorTeamShmemSize< typename DriverType::functor_type >::value( f , blockSize ); + cudaOccupancyMaxActiveBlocksPerMultiprocessor( + &numBlocks, + cuda_parallel_launch_constant_memory, + blockSize, + sharedmem); + + while (blockSize<1024 && numBlocks>0) { + blockSize*=2; + sharedmem = FunctorTeamShmemSize< typename DriverType::functor_type >::value( f , blockSize ); + + cudaOccupancyMaxActiveBlocksPerMultiprocessor( + &numBlocks, + cuda_parallel_launch_constant_memory, + blockSize, + sharedmem); + } + if(numBlocks>0) return blockSize; + else return blockSize/2; + } else { + int blockSize=32; + int sharedmem = FunctorTeamShmemSize< typename DriverType::functor_type >::value( f , blockSize ); + cudaOccupancyMaxActiveBlocksPerMultiprocessor( + &numBlocks, + cuda_parallel_launch_local_memory, + blockSize, + sharedmem); + + while (blockSize<1024 && numBlocks>0) { + blockSize*=2; + sharedmem = FunctorTeamShmemSize< typename DriverType::functor_type >::value( f , blockSize ); + + cudaOccupancyMaxActiveBlocksPerMultiprocessor( + &numBlocks, + cuda_parallel_launch_local_memory, + blockSize, + sharedmem); + } + if(numBlocks>0) return blockSize; + else return blockSize/2; + } +#endif +} + +template +int cuda_get_opt_block_size(const typename DriverType::functor_type & f) { +#if ( CUDA_VERSION < 6050 ) + return 256; +#else + bool Large = ( CudaTraits::ConstantMemoryUseThreshold < sizeof(DriverType) ); + + int blockSize=16; + int numBlocks; + int sharedmem; + int maxOccupancy=0; + int bestBlockSize=0; + + if(Large) { + while(blockSize<1024) { + blockSize*=2; + + //calculate the occupancy with that optBlockSize and check whether its larger than the largest one found so far + sharedmem = FunctorTeamShmemSize< typename DriverType::functor_type >::value( f , blockSize ); + cudaOccupancyMaxActiveBlocksPerMultiprocessor( + &numBlocks, + cuda_parallel_launch_constant_memory, + blockSize, + sharedmem); + if(maxOccupancy < numBlocks*blockSize) { + maxOccupancy = numBlocks*blockSize; + bestBlockSize = blockSize; + } + } + } else { + while(blockSize<1024) { + blockSize*=2; + sharedmem = FunctorTeamShmemSize< typename DriverType::functor_type >::value( f , blockSize ); + + cudaOccupancyMaxActiveBlocksPerMultiprocessor( + &numBlocks, + cuda_parallel_launch_local_memory, + blockSize, + sharedmem); + + if(maxOccupancy < numBlocks*blockSize) { + maxOccupancy = numBlocks*blockSize; + bestBlockSize = blockSize; + } + } + } + return bestBlockSize; +#endif +} + +} +} + +#define CUDA_SAFE_CALL( call ) \ + Kokkos::Impl::cuda_internal_safe_call( call , #call, __FILE__, __LINE__ ) + +#endif /* #ifndef KOKKOS_CUDA_INTERNAL_HPP */ + diff --git a/lib/kokkos/core/src/Cuda/Kokkos_Cuda_Parallel.hpp b/lib/kokkos/core/src/Cuda/Kokkos_Cuda_Parallel.hpp new file mode 100755 index 0000000000..1faf52ba61 --- /dev/null +++ b/lib/kokkos/core/src/Cuda/Kokkos_Cuda_Parallel.hpp @@ -0,0 +1,1591 @@ +/* +//@HEADER +// ************************************************************************ +// +// Kokkos: Manycore Performance-Portable Multidimensional Arrays +// Copyright (2012) Sandia Corporation +// +// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +// the U.S. Government retains certain rights in this software. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// 1. Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// 3. Neither the name of the Corporation nor the names of the +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Questions? Contact H. Carter Edwards (hcedwar@sandia.gov) +// +// ************************************************************************ +//@HEADER +*/ + +#ifndef KOKKOS_CUDA_PARALLEL_HPP +#define KOKKOS_CUDA_PARALLEL_HPP + +#include +#include + +#if defined( __CUDACC__ ) + +#include +#include + +#include +#include +#include +#include +//---------------------------------------------------------------------------- +//---------------------------------------------------------------------------- + +namespace Kokkos { +namespace Impl { + +template< typename Type > +struct CudaJoinFunctor { + typedef Type value_type ; + + KOKKOS_INLINE_FUNCTION + static void join( volatile value_type & update , + volatile const value_type & input ) + { update += input ; } +}; + +class CudaTeamMember { +private: + + typedef Kokkos::Cuda execution_space ; + typedef execution_space::scratch_memory_space scratch_memory_space ; + + void * m_team_reduce ; + scratch_memory_space m_team_shared ; + int m_league_rank ; + int m_league_size ; + +public: + +#if defined( __CUDA_ARCH__ ) + + __device__ inline + const execution_space::scratch_memory_space & team_shmem() const + { return m_team_shared ; } + + __device__ inline int league_rank() const { return m_league_rank ; } + __device__ inline int league_size() const { return m_league_size ; } + __device__ inline int team_rank() const { return threadIdx.y ; } + __device__ inline int team_size() const { return blockDim.y ; } + + __device__ inline void team_barrier() const { __syncthreads(); } + + template + __device__ inline void team_broadcast(ValueType& value, const int& thread_id) const { + __shared__ ValueType sh_val; + if(threadIdx.x == 0 && threadIdx.y == thread_id) { + sh_val = val; + } + team_barrier(); + val = sh_val; + } + +#ifdef KOKKOS_HAVE_CXX11 + template< class ValueType, class JoinOp > + __device__ inline + typename JoinOp::value_type team_reduce( const ValueType & value + , const JoinOp & op_in ) const + { + typedef JoinLambdaAdapter JoinOpFunctor ; + const JoinOpFunctor op(op_in); + ValueType * const base_data = (ValueType *) m_team_reduce ; +#else + template< class JoinOp > + __device__ inline + typename JoinOp::value_type team_reduce( const typename JoinOp::value_type & value + , const JoinOp & op ) const + { + typedef JoinOp JoinOpFunctor ; + typename JoinOp::value_type * const base_data = (typename JoinOp::value_type *) m_team_reduce ; +#endif + + __syncthreads(); // Don't write in to shared data until all threads have entered this function + + if ( 0 == threadIdx.y ) { base_data[0] = 0 ; } + + base_data[ threadIdx.y ] = value ; + + Impl::cuda_intra_block_reduce_scan( op , base_data ); + + return base_data[ blockDim.y - 1 ]; + } + + /** \brief Intra-team exclusive prefix sum with team_rank() ordering + * with intra-team non-deterministic ordering accumulation. + * + * The global inter-team accumulation value will, at the end of the + * league's parallel execution, be the scan's total. + * Parallel execution ordering of the league's teams is non-deterministic. + * As such the base value for each team's scan operation is similarly + * non-deterministic. + */ + template< typename Type > + __device__ inline Type team_scan( const Type & value , Type * const global_accum ) const + { + Type * const base_data = (Type *) m_team_reduce ; + + __syncthreads(); // Don't write in to shared data until all threads have entered this function + + if ( 0 == threadIdx.y ) { base_data[0] = 0 ; } + + base_data[ threadIdx.y + 1 ] = value ; + + Impl::cuda_intra_block_reduce_scan,void>( Impl::CudaJoinFunctor() , base_data + 1 ); + + if ( global_accum ) { + if ( blockDim.y == threadIdx.y + 1 ) { + base_data[ blockDim.y ] = atomic_fetch_add( global_accum , base_data[ blockDim.y ] ); + } + __syncthreads(); // Wait for atomic + base_data[ threadIdx.y ] += base_data[ blockDim.y ] ; + } + + return base_data[ threadIdx.y ]; + } + + /** \brief Intra-team exclusive prefix sum with team_rank() ordering. + * + * The highest rank thread can compute the reduction total as + * reduction_total = dev.team_scan( value ) + value ; + */ + template< typename Type > + __device__ inline Type team_scan( const Type & value ) const + { return this->template team_scan( value , 0 ); } + + +#ifdef KOKKOS_HAVE_CXX11 + template< class Operation > + __device__ inline void vector_single(const Operation & op) const { + if(threadIdx.x == 0) + op(); + } + + template< class Operation, typename ValueType> + __device__ inline void vector_single(const Operation & op, ValueType& bcast) const { + if(threadIdx.x == 0) + op(); + bcast = shfl(bcast,0,blockDim.x); + } + +#endif + + //---------------------------------------- + // Private for the driver + + __device__ inline + CudaTeamMember( void * shared + , const int shared_begin + , const int shared_size + , const int arg_league_rank + , const int arg_league_size ) + : m_team_reduce( shared ) + , m_team_shared( ((char *)shared) + shared_begin , shared_size ) + , m_league_rank( arg_league_rank ) + , m_league_size( arg_league_size ) + {} + +#else + + const execution_space::scratch_memory_space & team_shmem() const {return m_team_shared;} + + int league_rank() const {return 0;} + int league_size() const {return 1;} + int team_rank() const {return 0;} + int team_size() const {return 1;} + + void team_barrier() const {} + template + void team_broadcast(ValueType& value, const int& thread_id) const {} + + template< class JoinOp > + typename JoinOp::value_type team_reduce( const typename JoinOp::value_type & value + , const JoinOp & op ) const {return typename JoinOp::value_type();} + + template< typename Type > + Type team_scan( const Type & value , Type * const global_accum ) const {return Type();} + + template< typename Type > + Type team_scan( const Type & value ) const {return Type();} + +#ifdef KOKKOS_HAVE_CXX11 + template< class Operation > + void vector_single(const Operation & op) const {} + + template< class Operation , typename ValueType> + void vector_single(const Operation & op, ValueType& val) const {} +#endif + //---------------------------------------- + // Private for the driver + + CudaTeamMember( void * shared + , const int shared_begin + , const int shared_end + , const int arg_league_rank + , const int arg_league_size ); + +#endif /* #if ! defined( __CUDA_ARCH__ ) */ + +}; + +} // namespace Impl + +template< class Arg0 , class Arg1 > +class TeamPolicy< Arg0 , Arg1 , Kokkos::Cuda > +{ +private: + + enum { MAX_WARP = 8 }; + + const int m_league_size ; + const int m_team_size ; + const int m_vector_length ; + +public: + + //! Tag this class as a kokkos execution policy + typedef TeamPolicy execution_policy ; + + //! Execution space of this execution policy + typedef Kokkos::Cuda execution_space ; + + typedef typename + Impl::if_c< ! Impl::is_same< Kokkos::Cuda , Arg0 >::value , Arg0 , Arg1 >::type + work_tag ; + + //---------------------------------------- + + template< class FunctorType > + inline static + int team_size_max( const FunctorType & functor ) + { + int n = MAX_WARP * Impl::CudaTraits::WarpSize ; + + for ( ; n ; n >>= 1 ) { + const int shmem_size = + /* for global reduce */ Impl::cuda_single_inter_block_reduce_scan_shmem( functor , n ) + /* for team reduce */ + ( n + 2 ) * sizeof(double) + /* for team shared */ + Impl::FunctorTeamShmemSize< FunctorType >::value( functor , n ); + + if ( shmem_size < Impl::CudaTraits::SharedMemoryCapacity ) break ; + } + + return n ; + } + + template< class FunctorType > + static int team_size_recommended( const FunctorType & functor ) + { return team_size_max( functor ); } + + inline static + int vector_length_max() + { return Impl::CudaTraits::WarpSize; } + + //---------------------------------------- + + inline int vector_length() const { return m_vector_length ; } + inline int team_size() const { return m_team_size ; } + inline int league_size() const { return m_league_size ; } + + /** \brief Specify league size, request team size */ + TeamPolicy( execution_space & , int league_size , int team_size_request , int vector_length_request = 1 ) + : m_league_size( league_size ) + , m_team_size( team_size_request ) + , m_vector_length ( vector_length_request ) + { + // Allow only power-of-two vector_length + int check = 0; + for(int k = 1; k < vector_length_max(); k*=2) + if(k == vector_length_request) + check = 1; + if(!check) + Impl::throw_runtime_exception( "Requested non-power-of-two vector length for TeamPolicy."); + + // Make sure league size is permissable + if(league_size >= int(Impl::cuda_internal_maximum_grid_count())) + Impl::throw_runtime_exception( "Requested too large league_size for TeamPolicy on Cuda execution space."); + } + + TeamPolicy( int league_size , int team_size_request , int vector_length_request = 1 ) + : m_league_size( league_size ) + , m_team_size( team_size_request ) + , m_vector_length ( vector_length_request ) + { + // Allow only power-of-two vector_length + int check = 0; + for(int k = 1; k < vector_length_max(); k*=2) + if(k == vector_length_request) + check = 1; + if(!check) + Impl::throw_runtime_exception( "Requested non-power-of-two vector length for TeamPolicy."); + + // Make sure league size is permissable + if(league_size >= int(Impl::cuda_internal_maximum_grid_count())) + Impl::throw_runtime_exception( "Requested too large league_size for TeamPolicy on Cuda execution space."); + + } + + typedef Kokkos::Impl::CudaTeamMember member_type ; +}; + +} // namespace Kokkos + +//---------------------------------------------------------------------------- +//---------------------------------------------------------------------------- + +namespace Kokkos { +namespace Impl { + +template< class FunctorType , class Arg0 , class Arg1 , class Arg2 > +class ParallelFor< FunctorType , Kokkos::RangePolicy< Arg0 , Arg1 , Arg2 , Kokkos::Cuda > > +{ +private: + + typedef Kokkos::RangePolicy< Arg0 , Arg1 , Arg2 , Kokkos::Cuda > Policy ; + + const FunctorType m_functor ; + const Policy m_policy ; + + ParallelFor(); + ParallelFor & operator = ( const ParallelFor & ); + + template< class Tag > + inline static + __device__ + void driver( const FunctorType & functor + , typename Impl::enable_if< Impl::is_same< Tag , void >::value + , typename Policy::member_type const & >::type iwork + ) + { functor( iwork ); } + + template< class Tag > + inline static + __device__ + void driver( const FunctorType & functor + , typename Impl::enable_if< ! Impl::is_same< Tag , void >::value + , typename Policy::member_type const & >::type iwork + ) + { functor( Tag() , iwork ); } + +public: + + typedef FunctorType functor_type ; + + inline + __device__ + void operator()(void) const + { + const typename Policy::member_type work_stride = blockDim.y * gridDim.x ; + const typename Policy::member_type work_end = m_policy.end(); + + for ( typename Policy::member_type + iwork = m_policy.begin() + threadIdx.y + blockDim.y * blockIdx.x ; + iwork < work_end ; + iwork += work_stride ) { + ParallelFor::template driver< typename Policy::work_tag >( m_functor, iwork ); + } + } + + ParallelFor( const FunctorType & functor , + const Policy & policy ) + : m_functor( functor ) + , m_policy( policy ) + { + const dim3 block( 1 , CudaTraits::WarpSize * cuda_internal_maximum_warp_count(), 1); + const dim3 grid( std::min( ( int( policy.end() - policy.begin() ) + block.y - 1 ) / block.y + , cuda_internal_maximum_grid_count() ) + , 1 , 1); + + CudaParallelLaunch< ParallelFor >( *this , grid , block , 0 ); + } +}; + +template< class FunctorType , class Arg0 , class Arg1 > +class ParallelFor< FunctorType , Kokkos::TeamPolicy< Arg0 , Arg1 , Kokkos::Cuda > > +{ +private: + + typedef Kokkos::TeamPolicy< Arg0 , Arg1 , Kokkos::Cuda > Policy ; + +public: + + typedef FunctorType functor_type ; + typedef Cuda::size_type size_type ; + +private: + + // Algorithmic constraints: blockDim.y is a power of two AND blockDim.y == blockDim.z == 1 + // shared memory utilization: + // + // [ team reduce space ] + // [ team shared space ] + // + + const FunctorType m_functor ; + size_type m_shmem_begin ; + size_type m_shmem_size ; + size_type m_league_size ; + + template< class TagType > + KOKKOS_FORCEINLINE_FUNCTION + void driver( typename Impl::enable_if< Impl::is_same< TagType , void >::value , + const typename Policy::member_type & >::type member ) const + { m_functor( member ); } + + template< class TagType > + KOKKOS_FORCEINLINE_FUNCTION + void driver( typename Impl::enable_if< ! Impl::is_same< TagType , void >::value , + const typename Policy::member_type & >::type member ) const + { m_functor( TagType() , member ); } + +public: + + __device__ inline + void operator()(void) const + { + // Iterate this block through the league + for ( int league_rank = blockIdx.x ; league_rank < m_league_size ; league_rank += gridDim.x ) { + + ParallelFor::template driver< typename Policy::work_tag >( + typename Policy::member_type( kokkos_impl_cuda_shared_memory() + , m_shmem_begin + , m_shmem_size + , league_rank + , m_league_size ) ); + } + } + + + ParallelFor( const FunctorType & functor + , const Policy & policy + ) + : m_functor( functor ) + , m_shmem_begin( sizeof(double) * ( policy.team_size() + 2 ) ) + , m_shmem_size( FunctorTeamShmemSize< FunctorType >::value( functor , policy.team_size() ) ) + , m_league_size( policy.league_size() ) + { + // Functor's reduce memory, team scan memory, and team shared memory depend upon team size. + + const int shmem_size_total = m_shmem_begin + m_shmem_size ; + + if ( CudaTraits::SharedMemoryCapacity < shmem_size_total ) { + Kokkos::Impl::throw_runtime_exception(std::string("Kokkos::Impl::ParallelFor< Cuda > insufficient shared memory")); + } + + const dim3 grid( int(policy.league_size()) , 1 , 1 ); + const dim3 block( policy.vector_length() , policy.team_size() , 1 ); + + CudaParallelLaunch< ParallelFor >( *this, grid, block, shmem_size_total ); // copy to device and execute + } +}; + +} // namespace Impl +} // namespace Kokkos + +//---------------------------------------------------------------------------- +//---------------------------------------------------------------------------- + +namespace Kokkos { +namespace Impl { + +template< class FunctorType , class Arg0 , class Arg1 , class Arg2 > +class ParallelReduce< FunctorType , Kokkos::RangePolicy< Arg0 , Arg1 , Arg2 , Kokkos::Cuda > > +{ +private: + + typedef Kokkos::RangePolicy Policy ; + typedef typename Policy::WorkRange work_range ; + typedef typename Policy::work_tag work_tag ; + typedef Kokkos::Impl::FunctorValueTraits< FunctorType , work_tag > ValueTraits ; + typedef Kokkos::Impl::FunctorValueInit< FunctorType , work_tag > ValueInit ; + +public: + + typedef typename ValueTraits::pointer_type pointer_type ; + typedef typename ValueTraits::value_type value_type ; + typedef typename ValueTraits::reference_type reference_type ; + typedef FunctorType functor_type ; + typedef Cuda::size_type size_type ; + + // Algorithmic constraints: blockSize is a power of two AND blockDim.y == blockDim.z == 1 + + const FunctorType m_functor ; + const Policy m_policy ; + size_type * m_scratch_space ; + size_type * m_scratch_flags ; + size_type * m_unified_space ; + + // Determine block size constrained by shared memory: + static inline + unsigned local_block_size( const FunctorType & f ) + { + unsigned n = CudaTraits::WarpSize * 8 ; + while ( n && CudaTraits::SharedMemoryCapacity < cuda_single_inter_block_reduce_scan_shmem( f , n ) ) { n >>= 1 ; } + return n ; + } + + template< class Tag > + inline static + __device__ + void driver( const FunctorType & functor + , typename Impl::enable_if< Impl::is_same< Tag , void >::value + , typename Policy::member_type const & >::type iwork + , reference_type value ) + { functor( iwork , value ); } + + template< class Tag > + inline static + __device__ + void driver( const FunctorType & functor + , typename Impl::enable_if< ! Impl::is_same< Tag , void >::value + , typename Policy::member_type const & >::type iwork + , reference_type value ) + { functor( Tag() , iwork , value ); } + +#ifndef KOKKOS_EXPERIMENTAL_CUDA_SHFL_REDUCTION + __device__ inline + void operator()(void) const + { + const integral_nonzero_constant< size_type , ValueTraits::StaticValueSize / sizeof(size_type) > + word_count( ValueTraits::value_size( m_functor ) / sizeof(size_type) ); + + { + reference_type value = + ValueInit::init( m_functor , kokkos_impl_cuda_shared_memory() + threadIdx.y * word_count.value ); + + // Number of blocks is bounded so that the reduction can be limited to two passes. + // Each thread block is given an approximately equal amount of work to perform. + // Accumulate the values for this block. + // The accumulation ordering does not match the final pass, but is arithmatically equivalent. + + const work_range range( m_policy , blockIdx.x , gridDim.x ); + + for ( typename work_range::member_type iwork = range.begin() + threadIdx.y , iwork_end = range.end() ; + iwork < iwork_end ; iwork += blockDim.y ) { + ParallelReduce::template driver< work_tag >( m_functor , iwork , value ); + } + } + + // Reduce with final value at blockDim.y - 1 location. + if ( cuda_single_inter_block_reduce_scan( + m_functor , blockIdx.x , gridDim.x , + kokkos_impl_cuda_shared_memory() , m_scratch_space , m_scratch_flags ) ) { + + // This is the final block with the final result at the final threads' location + + size_type * const shared = kokkos_impl_cuda_shared_memory() + ( blockDim.y - 1 ) * word_count.value ; + size_type * const global = m_unified_space ? m_unified_space : m_scratch_space ; + + if ( threadIdx.y == 0 ) { + Kokkos::Impl::FunctorFinal< FunctorType , work_tag >::final( m_functor , shared ); + } + + if ( CudaTraits::WarpSize < word_count.value ) { __syncthreads(); } + + for ( unsigned i = threadIdx.y ; i < word_count.value ; i += blockDim.y ) { global[i] = shared[i]; } + } + } +#else + __device__ inline + void operator()(void) const + { + + value_type value = 0; + + // Number of blocks is bounded so that the reduction can be limited to two passes. + // Each thread block is given an approximately equal amount of work to perform. + // Accumulate the values for this block. + // The accumulation ordering does not match the final pass, but is arithmatically equivalent. + + const Policy range( m_policy , blockIdx.x , gridDim.x ); + + for ( typename Policy::member_type iwork = range.begin() + threadIdx.y , iwork_end = range.end() ; + iwork < iwork_end ; iwork += blockDim.y ) { + ParallelReduce::template driver< work_tag >( m_functor , iwork , value ); + } + + pointer_type const result = (pointer_type) (m_unified_space ? m_unified_space : m_scratch_space) ; + int max_active_thread = range.end()-range.begin() < blockDim.y ? range.end() - range.begin():blockDim.y; + max_active_thread = max_active_thread == 0?blockDim.y:max_active_thread; + if(Impl::cuda_inter_block_reduction > + (value,Impl::JoinAdd(),m_scratch_space,result,m_scratch_flags,max_active_thread)) { + const unsigned id = threadIdx.y*blockDim.x + threadIdx.x; + if(id==0) { + Kokkos::Impl::FunctorFinal< FunctorType , work_tag >::final( m_functor , (void*) &value ); + *result = value; + } + } + } +#endif + template< class HostViewType > + ParallelReduce( const FunctorType & functor + , const Policy & policy + , const HostViewType & result + ) + : m_functor( functor ) + , m_policy( policy ) + , m_scratch_space( 0 ) + , m_scratch_flags( 0 ) + , m_unified_space( 0 ) + { + const int block_size = local_block_size( functor ); + const int block_count = std::min( int(block_size) + , ( int(policy.end() - policy.begin()) + block_size - 1 ) / block_size + ); + + m_scratch_space = cuda_internal_scratch_space( ValueTraits::value_size( functor ) * block_count ); + m_scratch_flags = cuda_internal_scratch_flags( sizeof(size_type) ); + m_unified_space = cuda_internal_scratch_unified( ValueTraits::value_size( functor ) ); + + const dim3 grid( block_count , 1 , 1 ); + const dim3 block( 1 , block_size , 1 ); // REQUIRED DIMENSIONS ( 1 , N , 1 ) +#ifdef KOKKOS_EXPERIMENTAL_CUDA_SHFL_REDUCTION + const int shmem = 0; +#else + const int shmem = cuda_single_inter_block_reduce_scan_shmem( m_functor , block.y ); +#endif + + CudaParallelLaunch< ParallelReduce >( *this, grid, block, shmem ); // copy to device and execute + + Cuda::fence(); + + if ( result.ptr_on_device() ) { + if ( m_unified_space ) { + const int count = ValueTraits::value_count( m_functor ); + for ( int i = 0 ; i < count ; ++i ) { result.ptr_on_device()[i] = pointer_type(m_unified_space)[i] ; } + } + else { + const int size = ValueTraits::value_size( m_functor ); + DeepCopy( result.ptr_on_device() , m_scratch_space , size ); + } + } + } +}; + +template< class FunctorType , class Arg0 , class Arg1 > +class ParallelReduce< FunctorType , Kokkos::TeamPolicy< Arg0 , Arg1 , Kokkos::Cuda > > +{ +private: + + typedef Kokkos::TeamPolicy Policy ; + typedef typename Policy::work_tag work_tag ; + typedef Kokkos::Impl::FunctorValueTraits< FunctorType , work_tag > ValueTraits ; + typedef Kokkos::Impl::FunctorValueInit< FunctorType , work_tag > ValueInit ; + typedef typename ValueTraits::pointer_type pointer_type ; + typedef typename ValueTraits::reference_type reference_type ; + +public: + + typedef FunctorType functor_type ; + typedef Cuda::size_type size_type ; + +private: + + // Algorithmic constraints: blockDim.y is a power of two AND blockDim.y == blockDim.z == 1 + // shared memory utilization: + // + // [ global reduce space ] + // [ team reduce space ] + // [ team shared space ] + // + + const FunctorType m_functor ; + size_type * m_scratch_space ; + size_type * m_scratch_flags ; + size_type * m_unified_space ; + size_type m_team_begin ; + size_type m_shmem_begin ; + size_type m_shmem_size ; + size_type m_league_size ; + + template< class TagType > + KOKKOS_FORCEINLINE_FUNCTION + void driver( typename Impl::enable_if< Impl::is_same< TagType , void >::value , + const typename Policy::member_type & >::type member + , reference_type update ) const + { m_functor( member , update ); } + + template< class TagType > + KOKKOS_FORCEINLINE_FUNCTION + void driver( typename Impl::enable_if< ! Impl::is_same< TagType , void >::value , + const typename Policy::member_type & >::type member + , reference_type update ) const + { m_functor( TagType() , member , update ); } + +public: + + __device__ inline + void operator()(void) const + { + const integral_nonzero_constant< size_type , ValueTraits::StaticValueSize / sizeof(size_type) > + word_count( ValueTraits::value_size( m_functor ) / sizeof(size_type) ); + + reference_type value = + ValueInit::init( m_functor , kokkos_impl_cuda_shared_memory() + threadIdx.y * word_count.value ); + + // Iterate this block through the league + for ( int league_rank = blockIdx.x ; league_rank < m_league_size ; league_rank += gridDim.x ) { + + ParallelReduce::template driver< work_tag > + ( typename Policy::member_type( kokkos_impl_cuda_shared_memory() + m_team_begin + , m_shmem_begin + , m_shmem_size + , league_rank + , m_league_size ) + , value ); + } + + // Reduce with final value at blockDim.y - 1 location. + if ( cuda_single_inter_block_reduce_scan( + m_functor , blockIdx.x , gridDim.x , + kokkos_impl_cuda_shared_memory() , m_scratch_space , m_scratch_flags ) ) { + + // This is the final block with the final result at the final threads' location + + size_type * const shared = kokkos_impl_cuda_shared_memory() + ( blockDim.y - 1 ) * word_count.value ; + size_type * const global = m_unified_space ? m_unified_space : m_scratch_space ; + + if ( threadIdx.y == 0 ) { + Kokkos::Impl::FunctorFinal< FunctorType , work_tag >::final( m_functor , shared ); + } + + if ( CudaTraits::WarpSize < word_count.value ) { __syncthreads(); } + + for ( unsigned i = threadIdx.y ; i < word_count.value ; i += blockDim.y ) { global[i] = shared[i]; } + } + } + + + template< class HostViewType > + ParallelReduce( const FunctorType & functor + , const Policy & policy + , const HostViewType & result + ) + : m_functor( functor ) + , m_scratch_space( 0 ) + , m_scratch_flags( 0 ) + , m_unified_space( 0 ) + , m_team_begin( cuda_single_inter_block_reduce_scan_shmem( functor , policy.team_size() ) ) + , m_shmem_begin( sizeof(double) * ( policy.team_size() + 2 ) ) + , m_shmem_size( FunctorTeamShmemSize< FunctorType >::value( functor , policy.team_size() ) ) + , m_league_size( policy.league_size() ) + { + + // The global parallel_reduce does not support vector_length other than 1 at the moment + if(policy.vector_length() > 1) + Impl::throw_runtime_exception( "Kokkos::parallel_reduce with a TeamPolicy using a vector length of greater than 1 is not currently supported for CUDA."); + + // Functor's reduce memory, team scan memory, and team shared memory depend upon team size. + + const int shmem_size_total = m_team_begin + m_shmem_begin + m_shmem_size ; + const int not_power_of_two = 0 != ( policy.team_size() & ( policy.team_size() - 1 ) ); + + if ( not_power_of_two || CudaTraits::SharedMemoryCapacity < shmem_size_total ) { + Kokkos::Impl::throw_runtime_exception(std::string("Kokkos::Impl::ParallelReduce< Cuda > bad team size")); + } + + const int block_count = std::min( policy.league_size() , policy.team_size() ); + + m_scratch_space = cuda_internal_scratch_space( ValueTraits::value_size( functor ) * block_count ); + m_scratch_flags = cuda_internal_scratch_flags( sizeof(size_type) ); + m_unified_space = cuda_internal_scratch_unified( ValueTraits::value_size( functor ) ); + + const dim3 grid( block_count , 1 , 1 ); + const dim3 block( 1 , policy.team_size() , 1 ); // REQUIRED DIMENSIONS ( 1 , N , 1 ) + + CudaParallelLaunch< ParallelReduce >( *this, grid, block, shmem_size_total ); // copy to device and execute + + Cuda::fence(); + + if ( result.ptr_on_device() ) { + if ( m_unified_space ) { + const int count = ValueTraits::value_count( m_functor ); + for ( int i = 0 ; i < count ; ++i ) { result.ptr_on_device()[i] = pointer_type(m_unified_space)[i] ; } + } + else { + const int size = ValueTraits::value_size( m_functor ); + DeepCopy( result.ptr_on_device() , m_scratch_space , size ); + } + } + } +}; + +} // namespace Impl +} // namespace Kokkos + +//---------------------------------------------------------------------------- +//---------------------------------------------------------------------------- + +namespace Kokkos { +namespace Impl { + +template< class FunctorType , class Arg0 , class Arg1 , class Arg2 > +class ParallelScan< FunctorType , Kokkos::RangePolicy< Arg0 , Arg1 , Arg2 , Kokkos::Cuda > > +{ +private: + + typedef Kokkos::RangePolicy Policy ; + typedef typename Policy::WorkRange work_range ; + typedef typename Policy::work_tag work_tag ; + typedef Kokkos::Impl::FunctorValueTraits< FunctorType , work_tag > ValueTraits ; + typedef Kokkos::Impl::FunctorValueInit< FunctorType , work_tag > ValueInit ; + typedef Kokkos::Impl::FunctorValueOps< FunctorType , work_tag > ValueOps ; + +public: + typedef typename ValueTraits::pointer_type pointer_type ; + typedef typename ValueTraits::reference_type reference_type ; + typedef FunctorType functor_type ; + typedef Cuda::size_type size_type ; + + // Algorithmic constraints: + // (a) blockDim.y is a power of two + // (b) blockDim.y == blockDim.z == 1 + // (c) gridDim.x <= blockDim.y * blockDim.y + // (d) gridDim.y == gridDim.z == 1 + + // Determine block size constrained by shared memory: + static inline + unsigned local_block_size( const FunctorType & f ) + { + // blockDim.y must be power of two = 128 (4 warps) or 256 (8 warps) or 512 (16 warps) + // gridDim.x <= blockDim.y * blockDim.y + // + // 4 warps was 10% faster than 8 warps and 20% faster than 16 warps in unit testing + + unsigned n = CudaTraits::WarpSize * 4 ; + while ( n && CudaTraits::SharedMemoryCapacity < cuda_single_inter_block_reduce_scan_shmem( f , n ) ) { n >>= 1 ; } + return n ; + } + + const FunctorType m_functor ; + const Policy m_policy ; + size_type * m_scratch_space ; + size_type * m_scratch_flags ; + size_type m_final ; + + template< class Tag > + inline static + __device__ + void driver( const FunctorType & functor + , typename Impl::enable_if< Impl::is_same< Tag , void >::value + , typename Policy::member_type const & >::type iwork + , reference_type value + , const bool final ) + { functor( iwork , value , final ); } + + template< class Tag > + inline static + __device__ + void driver( const FunctorType & functor + , typename Impl::enable_if< ! Impl::is_same< Tag , void >::value + , typename Policy::member_type const & >::type iwork + , reference_type value + , const bool final ) + { functor( Tag() , iwork , value , final ); } + + //---------------------------------------- + + __device__ inline + void initial(void) const + { + const integral_nonzero_constant< size_type , ValueTraits::StaticValueSize / sizeof(size_type) > + word_count( ValueTraits::value_size( m_functor ) / sizeof(size_type) ); + + size_type * const shared_value = kokkos_impl_cuda_shared_memory() + word_count.value * threadIdx.y ; + + ValueInit::init( m_functor , shared_value ); + + // Number of blocks is bounded so that the reduction can be limited to two passes. + // Each thread block is given an approximately equal amount of work to perform. + // Accumulate the values for this block. + // The accumulation ordering does not match the final pass, but is arithmatically equivalent. + + const work_range range( m_policy , blockIdx.x , gridDim.x ); + + for ( typename Policy::member_type iwork = range.begin() + threadIdx.y , iwork_end = range.end() ; + iwork < iwork_end ; iwork += blockDim.y ) { + ParallelScan::template driver< work_tag > + ( m_functor , iwork , ValueOps::reference( shared_value ) , false ); + } + + // Reduce and scan, writing out scan of blocks' totals and block-groups' totals. + // Blocks' scan values are written to 'blockIdx.x' location. + // Block-groups' scan values are at: i = ( j * blockDim.y - 1 ) for i < gridDim.x + cuda_single_inter_block_reduce_scan( m_functor , blockIdx.x , gridDim.x , kokkos_impl_cuda_shared_memory() , m_scratch_space , m_scratch_flags ); + } + + //---------------------------------------- + + __device__ inline + void final(void) const + { + const integral_nonzero_constant< size_type , ValueTraits::StaticValueSize / sizeof(size_type) > + word_count( ValueTraits::value_size( m_functor ) / sizeof(size_type) ); + + // Use shared memory as an exclusive scan: { 0 , value[0] , value[1] , value[2] , ... } + size_type * const shared_data = kokkos_impl_cuda_shared_memory(); + size_type * const shared_prefix = shared_data + word_count.value * threadIdx.y ; + size_type * const shared_accum = shared_data + word_count.value * ( blockDim.y + 1 ); + + // Starting value for this thread block is the previous block's total. + if ( blockIdx.x ) { + size_type * const block_total = m_scratch_space + word_count.value * ( blockIdx.x - 1 ); + for ( unsigned i = threadIdx.y ; i < word_count.value ; ++i ) { shared_accum[i] = block_total[i] ; } + } + else if ( 0 == threadIdx.y ) { + ValueInit::init( m_functor , shared_accum ); + } + + const work_range range( m_policy , blockIdx.x , gridDim.x ); + + for ( typename Policy::member_type iwork_base = range.begin(); iwork_base < range.end() ; iwork_base += blockDim.y ) { + + const typename Policy::member_type iwork = iwork_base + threadIdx.y ; + + __syncthreads(); // Don't overwrite previous iteration values until they are used + + ValueInit::init( m_functor , shared_prefix + word_count.value ); + + // Copy previous block's accumulation total into thread[0] prefix and inclusive scan value of this block + for ( unsigned i = threadIdx.y ; i < word_count.value ; ++i ) { + shared_data[i + word_count.value] = shared_data[i] = shared_accum[i] ; + } + + if ( CudaTraits::WarpSize < word_count.value ) { __syncthreads(); } // Protect against large scan values. + + // Call functor to accumulate inclusive scan value for this work item + if ( iwork < range.end() ) { + ParallelScan::template driver< work_tag > + ( m_functor , iwork , ValueOps::reference( shared_prefix + word_count.value ) , false ); + } + + // Scan block values into locations shared_data[1..blockDim.y] + cuda_intra_block_reduce_scan( m_functor , ValueTraits::pointer_type(shared_data+word_count.value) ); + + { + size_type * const block_total = shared_data + word_count.value * blockDim.y ; + for ( unsigned i = threadIdx.y ; i < word_count.value ; ++i ) { shared_accum[i] = block_total[i]; } + } + + // Call functor with exclusive scan value + if ( iwork < range.end() ) { + ParallelScan::template driver< work_tag > + ( m_functor , iwork , ValueOps::reference( shared_prefix ) , true ); + } + } + } + + //---------------------------------------- + + __device__ inline + void operator()(void) const + { + if ( ! m_final ) { + initial(); + } + else { + final(); + } + } + + ParallelScan( const FunctorType & functor , + const Policy & policy ) + : m_functor( functor ) + , m_policy( policy ) + , m_scratch_space( 0 ) + , m_scratch_flags( 0 ) + , m_final( false ) + { + enum { GridMaxComputeCapability_2x = 0x0ffff }; + + const int block_size = local_block_size( functor ); + + const int grid_max = ( block_size * block_size ) < GridMaxComputeCapability_2x ? + ( block_size * block_size ) : GridMaxComputeCapability_2x ; + + // At most 'max_grid' blocks: + const int nwork = policy.end() - policy.begin(); + const int max_grid = std::min( int(grid_max) , int(( nwork + block_size - 1 ) / block_size )); + + // How much work per block: + const int work_per_block = ( nwork + max_grid - 1 ) / max_grid ; + + // How many block are really needed for this much work: + const dim3 grid( ( nwork + work_per_block - 1 ) / work_per_block , 1 , 1 ); + const dim3 block( 1 , block_size , 1 ); // REQUIRED DIMENSIONS ( 1 , N , 1 ) + const int shmem = ValueTraits::value_size( functor ) * ( block_size + 2 ); + + m_scratch_space = cuda_internal_scratch_space( ValueTraits::value_size( functor ) * grid.x ); + m_scratch_flags = cuda_internal_scratch_flags( sizeof(size_type) * 1 ); + + m_final = false ; + CudaParallelLaunch< ParallelScan >( *this, grid, block, shmem ); // copy to device and execute + + m_final = true ; + CudaParallelLaunch< ParallelScan >( *this, grid, block, shmem ); // copy to device and execute + } + + void wait() const { Cuda::fence(); } +}; + +} // namespace Impl +} // namespace Kokkos + +//---------------------------------------------------------------------------- +//---------------------------------------------------------------------------- + +#ifdef KOKKOS_HAVE_CXX11 + +namespace Kokkos { +namespace Impl { + template + struct TeamThreadLoopBoundariesStruct { + typedef iType index_type; + const iType start; + const iType end; + const iType increment; + const CudaTeamMember& thread; + +#ifdef __CUDA_ARCH__ + __device__ inline + TeamThreadLoopBoundariesStruct (const CudaTeamMember& thread_, const iType& count): + start( threadIdx.y ), + end( count ), + increment( blockDim.y ), + thread(thread_) + {} +#else + KOKKOS_INLINE_FUNCTION + TeamThreadLoopBoundariesStruct (const CudaTeamMember& thread_, const iType& count): + start( 0 ), + end( count ), + increment( 1 ), + thread(thread_) + {} +#endif + }; + + template + struct ThreadVectorLoopBoundariesStruct { + typedef iType index_type; + const iType start; + const iType end; + const iType increment; + +#ifdef __CUDA_ARCH__ + __device__ inline + ThreadVectorLoopBoundariesStruct (const CudaTeamMember& thread, const iType& count): + start( threadIdx.x ), + end( count ), + increment( blockDim.x ) + {} +#else + KOKKOS_INLINE_FUNCTION + ThreadVectorLoopBoundariesStruct (const CudaTeamMember& thread_, const iType& count): + start( 0 ), + end( count ), + increment( 1 ) + {} +#endif + }; + +} // namespace Impl + +template +KOKKOS_INLINE_FUNCTION +Impl::TeamThreadLoopBoundariesStruct + TeamThreadLoop(const Impl::CudaTeamMember& thread, const iType& count) { + return Impl::TeamThreadLoopBoundariesStruct(thread,count); +} + +template +KOKKOS_INLINE_FUNCTION +Impl::ThreadVectorLoopBoundariesStruct + ThreadVectorLoop(Impl::CudaTeamMember thread, const iType count) { + return Impl::ThreadVectorLoopBoundariesStruct(thread,count); +} + +KOKKOS_INLINE_FUNCTION +Impl::ThreadSingleStruct PerTeam(const Impl::CudaTeamMember& thread) { + return Impl::ThreadSingleStruct(thread); +} + +KOKKOS_INLINE_FUNCTION +Impl::VectorSingleStruct PerThread(const Impl::CudaTeamMember& thread) { + return Impl::VectorSingleStruct(thread); +} + +} // namespace Kokkos + +namespace Kokkos { + + /** \brief Inter-thread parallel_for. Executes lambda(iType i) for each i=0..N-1. + * + * The range i=0..N-1 is mapped to all threads of the the calling thread team. + * This functionality requires C++11 support.*/ +template +KOKKOS_INLINE_FUNCTION +void parallel_for(const Impl::TeamThreadLoopBoundariesStruct& loop_boundaries, const Lambda& lambda) { + for( iType i = loop_boundaries.start; i < loop_boundaries.end; i+=loop_boundaries.increment) + lambda(i); +} + +/** \brief Inter-thread vector parallel_reduce. Executes lambda(iType i, ValueType & val) for each i=0..N-1. + * + * The range i=0..N-1 is mapped to all threads of the the calling thread team and a summation of + * val is performed and put into result. This functionality requires C++11 support.*/ +template< typename iType, class Lambda, typename ValueType > +KOKKOS_INLINE_FUNCTION +void parallel_reduce(const Impl::TeamThreadLoopBoundariesStruct& loop_boundaries, + const Lambda & lambda, ValueType& result) { + +#ifdef __CUDA_ARCH__ + result = ValueType(); + + for( iType i = loop_boundaries.start; i < loop_boundaries.end; i+=loop_boundaries.increment) { + lambda(i,result); + } + + Impl::cuda_intra_warp_reduction(result,[&] (ValueType& dst, const ValueType& src) { dst+=src; }); + Impl::cuda_inter_warp_reduction(result,[&] (ValueType& dst, const ValueType& src) { dst+=src; }); + +#endif +} + +/** \brief Intra-thread vector parallel_reduce. Executes lambda(iType i, ValueType & val) for each i=0..N-1. + * + * The range i=0..N-1 is mapped to all vector lanes of the the calling thread and a reduction of + * val is performed using JoinType(ValueType& val, const ValueType& update) and put into init_result. + * The input value of init_result is used as initializer for temporary variables of ValueType. Therefore + * the input value should be the neutral element with respect to the join operation (e.g. '0 for +-' or + * '1 for *'). This functionality requires C++11 support.*/ +template< typename iType, class Lambda, typename ValueType, class JoinType > +KOKKOS_INLINE_FUNCTION +void parallel_reduce(const Impl::TeamThreadLoopBoundariesStruct& loop_boundaries, + const Lambda & lambda, const JoinType& join, ValueType& init_result) { + +#ifdef __CUDA_ARCH__ + ValueType result = init_result; + + for( iType i = loop_boundaries.start; i < loop_boundaries.end; i+=loop_boundaries.increment) { + lambda(i,result); + } + + Impl::cuda_intra_warp_reduction(result, join ); + Impl::cuda_inter_warp_reduction(result, join ); + + init_result = result; +#endif +} + +} //namespace Kokkos + +namespace Kokkos { +/** \brief Intra-thread vector parallel_for. Executes lambda(iType i) for each i=0..N-1. + * + * The range i=0..N-1 is mapped to all vector lanes of the the calling thread. + * This functionality requires C++11 support.*/ +template +KOKKOS_INLINE_FUNCTION +void parallel_for(const Impl::ThreadVectorLoopBoundariesStruct& + loop_boundaries, const Lambda& lambda) { + + for( iType i = loop_boundaries.start; i < loop_boundaries.end; i+=loop_boundaries.increment) + lambda(i); +} + +/** \brief Intra-thread vector parallel_reduce. Executes lambda(iType i, ValueType & val) for each i=0..N-1. + * + * The range i=0..N-1 is mapped to all vector lanes of the the calling thread and a summation of + * val is performed and put into result. This functionality requires C++11 support.*/ +template< typename iType, class Lambda, typename ValueType > +KOKKOS_INLINE_FUNCTION +void parallel_reduce(const Impl::ThreadVectorLoopBoundariesStruct& + loop_boundaries, const Lambda & lambda, ValueType& result) { +#ifdef __CUDA_ARCH__ + ValueType val = ValueType(); + + for( iType i = loop_boundaries.start; i < loop_boundaries.end; i+=loop_boundaries.increment) { + lambda(i,val); + } + + result = val; + + if (loop_boundaries.increment > 1) + result += shfl_down(result, 1,loop_boundaries.increment); + if (loop_boundaries.increment > 2) + result += shfl_down(result, 2,loop_boundaries.increment); + if (loop_boundaries.increment > 4) + result += shfl_down(result, 4,loop_boundaries.increment); + if (loop_boundaries.increment > 8) + result += shfl_down(result, 8,loop_boundaries.increment); + if (loop_boundaries.increment > 16) + result += shfl_down(result, 16,loop_boundaries.increment); + + result = shfl(result,0,loop_boundaries.increment); +#endif +} + +/** \brief Intra-thread vector parallel_reduce. Executes lambda(iType i, ValueType & val) for each i=0..N-1. + * + * The range i=0..N-1 is mapped to all vector lanes of the the calling thread and a reduction of + * val is performed using JoinType(ValueType& val, const ValueType& update) and put into init_result. + * The input value of init_result is used as initializer for temporary variables of ValueType. Therefore + * the input value should be the neutral element with respect to the join operation (e.g. '0 for +-' or + * '1 for *'). This functionality requires C++11 support.*/ +template< typename iType, class Lambda, typename ValueType, class JoinType > +KOKKOS_INLINE_FUNCTION +void parallel_reduce(const Impl::ThreadVectorLoopBoundariesStruct& + loop_boundaries, const Lambda & lambda, const JoinType& join, ValueType& init_result) { + +#ifdef __CUDA_ARCH__ + ValueType result = init_result; + + for( iType i = loop_boundaries.start; i < loop_boundaries.end; i+=loop_boundaries.increment) { + lambda(i,result); + } + + if (loop_boundaries.increment > 1) + join( result, shfl_down(result, 1,loop_boundaries.increment)); + if (loop_boundaries.increment > 2) + join( result, shfl_down(result, 2,loop_boundaries.increment)); + if (loop_boundaries.increment > 4) + join( result, shfl_down(result, 4,loop_boundaries.increment)); + if (loop_boundaries.increment > 8) + join( result, shfl_down(result, 8,loop_boundaries.increment)); + if (loop_boundaries.increment > 16) + join( result, shfl_down(result, 16,loop_boundaries.increment)); + + init_result = shfl(result,0,loop_boundaries.increment); +#endif +} + +/** \brief Intra-thread vector parallel exclusive prefix sum. Executes lambda(iType i, ValueType & val, bool final) + * for each i=0..N-1. + * + * The range i=0..N-1 is mapped to all vector lanes in the thread and a scan operation is performed. + * Depending on the target execution space the operator might be called twice: once with final=false + * and once with final=true. When final==true val contains the prefix sum value. The contribution of this + * "i" needs to be added to val no matter whether final==true or not. In a serial execution + * (i.e. team_size==1) the operator is only called once with final==true. Scan_val will be set + * to the final sum value over all vector lanes. + * This functionality requires C++11 support.*/ +template< typename iType, class FunctorType > +KOKKOS_INLINE_FUNCTION +void parallel_scan(const Impl::ThreadVectorLoopBoundariesStruct& + loop_boundaries, const FunctorType & lambda) { + +#ifdef __CUDA_ARCH__ + typedef Kokkos::Impl::FunctorValueTraits< FunctorType , void > ValueTraits ; + typedef typename ValueTraits::value_type value_type ; + + value_type scan_val = value_type(); + const int VectorLength = blockDim.x; + + iType loop_bound = ((loop_boundaries.end+VectorLength-1)/VectorLength) * VectorLength; + for(int _i = threadIdx.x; _i < loop_bound; _i += VectorLength) { + value_type val = value_type(); + if(_i 1) { + const value_type tmp2 = shfl_up(tmp, 1,VectorLength); + if(threadIdx.x > 0) + tmp+=tmp2; + } + if(threadIdx.x%VectorLength == 1) + result_i = tmp; + if (VectorLength > 3) { + const value_type tmp2 = shfl_up(tmp, 2,VectorLength); + if(threadIdx.x > 1) + tmp+=tmp2; + } + if ((threadIdx.x%VectorLength >= 2) && + (threadIdx.x%VectorLength < 4)) + result_i = tmp; + if (VectorLength > 7) { + const value_type tmp2 = shfl_up(tmp, 4,VectorLength); + if(threadIdx.x > 3) + tmp+=tmp2; + } + if ((threadIdx.x%VectorLength >= 4) && + (threadIdx.x%VectorLength < 8)) + result_i = tmp; + if (VectorLength > 15) { + const value_type tmp2 = shfl_up(tmp, 8,VectorLength); + if(threadIdx.x > 7) + tmp+=tmp2; + } + if ((threadIdx.x%VectorLength >= 8) && + (threadIdx.x%VectorLength < 16)) + result_i = tmp; + if (VectorLength > 31) { + const value_type tmp2 = shfl_up(tmp, 16,VectorLength); + if(threadIdx.x > 15) + tmp+=tmp2; + } + if (threadIdx.x%VectorLength >= 16) + result_i = tmp; + + val = scan_val + result_i - val; + scan_val += shfl(tmp,VectorLength-1,VectorLength); + if(_i +KOKKOS_INLINE_FUNCTION +void single(const Impl::VectorSingleStruct& , const FunctorType& lambda) { +#ifdef __CUDA_ARCH__ + if(threadIdx.x == 0) lambda(); +#endif +} + +template +KOKKOS_INLINE_FUNCTION +void single(const Impl::ThreadSingleStruct& , const FunctorType& lambda) { +#ifdef __CUDA_ARCH__ + if(threadIdx.x == 0 && threadIdx.y == 0) lambda(); +#endif +} + +template +KOKKOS_INLINE_FUNCTION +void single(const Impl::VectorSingleStruct& , const FunctorType& lambda, ValueType& val) { +#ifdef __CUDA_ARCH__ + if(threadIdx.x == 0) lambda(val); + val = shfl(val,0,blockDim.x); +#endif +} + +template +KOKKOS_INLINE_FUNCTION +void single(const Impl::ThreadSingleStruct& single_struct, const FunctorType& lambda, ValueType& val) { +#ifdef __CUDA_ARCH__ + if(threadIdx.x == 0 && threadIdx.y == 0) { + lambda(val); + } + single_struct.team_member.team_broadcast(val,0); +#endif +} + +} + +#endif // KOKKOS_HAVE_CXX11 + +namespace Kokkos { +template +struct Vectorization { + typedef Kokkos::TeamPolicy< Cuda > team_policy ; + typedef typename team_policy::member_type team_member ; + enum {increment = N}; + +#ifdef __CUDA_ARCH__ + KOKKOS_FORCEINLINE_FUNCTION + static int begin() { return threadIdx.y%N;} +#else + KOKKOS_FORCEINLINE_FUNCTION + static int begin() { return 0;} +#endif + + KOKKOS_FORCEINLINE_FUNCTION + static int thread_rank(const team_member &dev) { + return dev.team_rank()/increment; + } + + KOKKOS_FORCEINLINE_FUNCTION + static int team_rank(const team_member &dev) { + return dev.team_rank()/increment; + } + + KOKKOS_FORCEINLINE_FUNCTION + static int team_size(const team_member &dev) { + return dev.team_size()/increment; + } + + KOKKOS_FORCEINLINE_FUNCTION + static int global_thread_rank(const team_member &dev) { + return (dev.league_rank()*dev.team_size()+dev.team_rank())/increment; + } + + KOKKOS_FORCEINLINE_FUNCTION + static bool is_lane_0(const team_member &dev) { + return (dev.team_rank()%increment)==0; + } + + template + KOKKOS_INLINE_FUNCTION + static Scalar reduce(const Scalar& val) { + #ifdef __CUDA_ARCH__ + __shared__ Scalar result[256]; + Scalar myresult; + for(int k=0;k 0 && tid<256) { + result[tid] = val; + if ( (N > 1) && (tid%2==0) ) + result[tid] += result[tid+1]; + if ( (N > 2) && (tid%4==0) ) + result[tid] += result[tid+2]; + if ( (N > 4) && (tid%8==0) ) + result[tid] += result[tid+4]; + if ( (N > 8) && (tid%16==0) ) + result[tid] += result[tid+8]; + if ( (N > 16) && (tid%32==0) ) + result[tid] += result[tid+16]; + myresult = result[tid]; + } + if(blockDim.y>256) + __syncthreads(); + } + return myresult; + #else + return val; + #endif + } + +#ifdef __CUDA_ARCH__ + #if (__CUDA_ARCH__ >= 300) + KOKKOS_INLINE_FUNCTION + static int reduce(const int& val) { + int result = val; + if (N > 1) + result += shfl_down(result, 1,N); + if (N > 2) + result += shfl_down(result, 2,N); + if (N > 4) + result += shfl_down(result, 4,N); + if (N > 8) + result += shfl_down(result, 8,N); + if (N > 16) + result += shfl_down(result, 16,N); + return result; + } + + KOKKOS_INLINE_FUNCTION + static unsigned int reduce(const unsigned int& val) { + unsigned int result = val; + if (N > 1) + result += shfl_down(result, 1,N); + if (N > 2) + result += shfl_down(result, 2,N); + if (N > 4) + result += shfl_down(result, 4,N); + if (N > 8) + result += shfl_down(result, 8,N); + if (N > 16) + result += shfl_down(result, 16,N); + return result; + } + + KOKKOS_INLINE_FUNCTION + static long int reduce(const long int& val) { + long int result = val; + if (N > 1) + result += shfl_down(result, 1,N); + if (N > 2) + result += shfl_down(result, 2,N); + if (N > 4) + result += shfl_down(result, 4,N); + if (N > 8) + result += shfl_down(result, 8,N); + if (N > 16) + result += shfl_down(result, 16,N); + return result; + } + + KOKKOS_INLINE_FUNCTION + static unsigned long int reduce(const unsigned long int& val) { + unsigned long int result = val; + if (N > 1) + result += shfl_down(result, 1,N); + if (N > 2) + result += shfl_down(result, 2,N); + if (N > 4) + result += shfl_down(result, 4,N); + if (N > 8) + result += shfl_down(result, 8,N); + if (N > 16) + result += shfl_down(result, 16,N); + return result; + } + + KOKKOS_INLINE_FUNCTION + static float reduce(const float& val) { + float result = val; + if (N > 1) + result += shfl_down(result, 1,N); + if (N > 2) + result += shfl_down(result, 2,N); + if (N > 4) + result += shfl_down(result, 4,N); + if (N > 8) + result += shfl_down(result, 8,N); + if (N > 16) + result += shfl_down(result, 16,N); + return result; + } + + KOKKOS_INLINE_FUNCTION + static double reduce(const double& val) { + double result = val; + if (N > 1) + result += shfl_down(result, 1,N); + if (N > 2) + result += shfl_down(result, 2,N); + if (N > 4) + result += shfl_down(result, 4,N); + if (N > 8) + result += shfl_down(result, 8,N); + if (N > 16) + result += shfl_down(result, 16,N); + return result; + } + #endif +#endif + +}; +} + +#endif /* defined( __CUDACC__ ) */ + +#endif /* #ifndef KOKKOS_CUDA_PARALLEL_HPP */ + diff --git a/lib/kokkos/core/src/Cuda/Kokkos_Cuda_ReduceScan.hpp b/lib/kokkos/core/src/Cuda/Kokkos_Cuda_ReduceScan.hpp new file mode 100755 index 0000000000..a723f629af --- /dev/null +++ b/lib/kokkos/core/src/Cuda/Kokkos_Cuda_ReduceScan.hpp @@ -0,0 +1,421 @@ +/* +//@HEADER +// ************************************************************************ +// +// Kokkos: Manycore Performance-Portable Multidimensional Arrays +// Copyright (2012) Sandia Corporation +// +// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +// the U.S. Government retains certain rights in this software. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// 1. Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// 3. Neither the name of the Corporation nor the names of the +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Questions? Contact H. Carter Edwards (hcedwar@sandia.gov) +// +// ************************************************************************ +//@HEADER +*/ + +#ifndef KOKKOS_CUDA_REDUCESCAN_HPP +#define KOKKOS_CUDA_REDUCESCAN_HPP + +#if defined( __CUDACC__ ) + +#include + +#include +#include +#include +#include +//---------------------------------------------------------------------------- +//---------------------------------------------------------------------------- + +namespace Kokkos { +namespace Impl { + + + +//Shfl based reductions +/* + * Algorithmic constraints: + * (a) threads with same threadIdx.y have same value + * (b) blockDim.x == power of two + * (c) blockDim.z == 1 + */ + +template< class ValueType , class JoinOp> +__device__ +inline void cuda_intra_warp_reduction( ValueType& result, + const JoinOp& join, + const int max_active_thread = blockDim.y) { + + unsigned int shift = 1; + + //Reduce over values from threads with different threadIdx.y + while(blockDim.x * shift < 32 ) { + const ValueType tmp = shfl_down(result, blockDim.x*shift,32u); + //Only join if upper thread is active (this allows non power of two for blockDim.y + if(threadIdx.y + shift < max_active_thread) + join(result , tmp); + shift*=2; + } + + result = shfl(result,0,32); +} + +template< class ValueType , class JoinOp> +__device__ +inline void cuda_inter_warp_reduction( ValueType& value, + const JoinOp& join, + const int max_active_thread = blockDim.y) { + + #define STEP_WIDTH 4 + __shared__ char sh_result[sizeof(ValueType)*STEP_WIDTH]; + ValueType* result = (ValueType*) & sh_result; + const unsigned step = 32 / blockDim.x; + unsigned shift = STEP_WIDTH; + const int id = threadIdx.y%step==0?threadIdx.y/step:65000; + if(id < STEP_WIDTH ) { + result[id] = value; + } + __syncthreads(); + while (shift<=max_active_thread/step) { + if(shift<=id && shift+STEP_WIDTH>id && threadIdx.x==0) { + join(result[id%STEP_WIDTH],value); + } + __syncthreads(); + shift+=STEP_WIDTH; + } + + + value = result[0]; + for(int i = 1; (i*step<=max_active_thread) && i +__device__ +inline void cuda_intra_block_reduction( ValueType& value, + const JoinOp& join, + const int max_active_thread = blockDim.y) { + cuda_intra_warp_reduction(value,join,max_active_thread); + cuda_inter_warp_reduction(value,join,max_active_thread); +} + +template< class FunctorType , class JoinOp> +__device__ +bool cuda_inter_block_reduction( typename FunctorValueTraits< FunctorType , void >::reference_type value, + const JoinOp& join, + Cuda::size_type * const m_scratch_space, + typename FunctorValueTraits< FunctorType , void >::pointer_type const result, + Cuda::size_type * const m_scratch_flags, + const int max_active_thread = blockDim.y) { + typedef typename FunctorValueTraits< FunctorType , void >::pointer_type pointer_type; + typedef typename FunctorValueTraits< FunctorType , void >::value_type value_type; + + //Do the intra-block reduction with shfl operations and static shared memory + cuda_intra_block_reduction(value,join,max_active_thread); + + const unsigned id = threadIdx.y*blockDim.x + threadIdx.x; + + //One thread in the block writes block result to global scratch_memory + if(id == 0 ) { + pointer_type global = ((pointer_type) m_scratch_space) + blockIdx.x; + *global = value; + } + + //One warp of last block performs inter block reduction through loading the block values from global scratch_memory + bool last_block = false; + + __syncthreads(); + if ( id < 32 ) { + Cuda::size_type count; + + //Figure out whether this is the last block + if(id == 0) + count = Kokkos::atomic_fetch_add(m_scratch_flags,1); + count = Kokkos::shfl(count,0,32); + + //Last block does the inter block reduction + if( count == gridDim.x - 1) { + //set flag back to zero + if(id == 0) + *m_scratch_flags = 0; + last_block = true; + value = 0; + + pointer_type const volatile global = (pointer_type) m_scratch_space ; + + //Reduce all global values with splitting work over threads in one warp + const int step_size = blockDim.x*blockDim.y < 32 ? blockDim.x*blockDim.y : 32; + for(int i=id; i 1) { + value_type tmp = Kokkos::shfl_down(value, 1,32); + if( id + 1 < gridDim.x ) + join(value, tmp); + } + if (blockDim.x*blockDim.y > 2) { + value_type tmp = Kokkos::shfl_down(value, 2,32); + if( id + 2 < gridDim.x ) + join(value, tmp); + } + if (blockDim.x*blockDim.y > 4) { + value_type tmp = Kokkos::shfl_down(value, 4,32); + if( id + 4 < gridDim.x ) + join(value, tmp); + } + if (blockDim.x*blockDim.y > 8) { + value_type tmp = Kokkos::shfl_down(value, 8,32); + if( id + 8 < gridDim.x ) + join(value, tmp); + } + if (blockDim.x*blockDim.y > 16) { + value_type tmp = Kokkos::shfl_down(value, 16,32); + if( id + 16 < gridDim.x ) + join(value, tmp); + } + } + } + + //The last block has in its thread=0 the global reduction value through "value" + return last_block; +} + +//---------------------------------------------------------------------------- +// See section B.17 of Cuda C Programming Guide Version 3.2 +// for discussion of +// __launch_bounds__(maxThreadsPerBlock,minBlocksPerMultiprocessor) +// function qualifier which could be used to improve performance. +//---------------------------------------------------------------------------- +// Maximize shared memory and minimize L1 cache: +// cudaFuncSetCacheConfig(MyKernel, cudaFuncCachePreferShared ); +// For 2.0 capability: 48 KB shared and 16 KB L1 +//---------------------------------------------------------------------------- +//---------------------------------------------------------------------------- +/* + * Algorithmic constraints: + * (a) blockDim.y is a power of two + * (b) blockDim.y <= 512 + * (c) blockDim.x == blockDim.z == 1 + */ + +template< bool DoScan , class FunctorType , class ArgTag > +__device__ +void cuda_intra_block_reduce_scan( const FunctorType & functor , + const typename FunctorValueTraits< FunctorType , ArgTag >::pointer_type base_data ) +{ + typedef FunctorValueTraits< FunctorType , ArgTag > ValueTraits ; + typedef FunctorValueJoin< FunctorType , ArgTag > ValueJoin ; + + typedef typename ValueTraits::pointer_type pointer_type ; + + const unsigned value_count = ValueTraits::value_count( functor ); + const unsigned BlockSizeMask = blockDim.y - 1 ; + + // Must have power of two thread count + + if ( BlockSizeMask & blockDim.y ) { Kokkos::abort("Cuda::cuda_intra_block_scan requires power-of-two blockDim"); } + +#define BLOCK_REDUCE_STEP( R , TD , S ) \ + if ( ! ( R & ((1<<(S+1))-1) ) ) { ValueJoin::join( functor , TD , (TD - (value_count< +__device__ +bool cuda_single_inter_block_reduce_scan( const FunctorType & functor , + const Cuda::size_type block_id , + const Cuda::size_type block_count , + Cuda::size_type * const shared_data , + Cuda::size_type * const global_data , + Cuda::size_type * const global_flags ) +{ + typedef Cuda::size_type size_type ; + typedef FunctorValueTraits< FunctorType , ArgTag > ValueTraits ; + typedef FunctorValueJoin< FunctorType , ArgTag > ValueJoin ; + typedef FunctorValueInit< FunctorType , ArgTag > ValueInit ; + typedef FunctorValueOps< FunctorType , ArgTag > ValueOps ; + + typedef typename ValueTraits::pointer_type pointer_type ; + typedef typename ValueTraits::reference_type reference_type ; + + const unsigned BlockSizeMask = blockDim.y - 1 ; + const unsigned BlockSizeShift = power_of_two_if_valid( blockDim.y ); + + // Must have power of two thread count + if ( BlockSizeMask & blockDim.y ) { Kokkos::abort("Cuda::cuda_single_inter_block_reduce_scan requires power-of-two blockDim"); } + + const integral_nonzero_constant< size_type , ValueTraits::StaticValueSize / sizeof(size_type) > + word_count( ValueTraits::value_size( functor ) / sizeof(size_type) ); + + // Reduce the accumulation for the entire block. + cuda_intra_block_reduce_scan( functor , pointer_type(shared_data) ); + + { + // Write accumulation total to global scratch space. + // Accumulation total is the last thread's data. + size_type * const shared = shared_data + word_count.value * BlockSizeMask ; + size_type * const global = global_data + word_count.value * block_id ; + + for ( size_type i = threadIdx.y ; i < word_count.value ; i += blockDim.y ) { global[i] = shared[i] ; } + } + + // Contributing blocks note that their contribution has been completed via an atomic-increment flag + // If this block is not the last block to contribute to this group then the block is done. + const bool is_last_block = + ! __syncthreads_or( threadIdx.y ? 0 : ( 1 + atomicInc( global_flags , block_count - 1 ) < block_count ) ); + + if ( is_last_block ) { + + const size_type b = ( long(block_count) * long(threadIdx.y) ) >> BlockSizeShift ; + const size_type e = ( long(block_count) * long( threadIdx.y + 1 ) ) >> BlockSizeShift ; + + { + void * const shared_ptr = shared_data + word_count.value * threadIdx.y ; + reference_type shared_value = ValueInit::init( functor , shared_ptr ); + + for ( size_type i = b ; i < e ; ++i ) { + ValueJoin::join( functor , shared_ptr , global_data + word_count.value * i ); + } + } + + cuda_intra_block_reduce_scan( functor , pointer_type(shared_data) ); + + if ( DoScan ) { + + size_type * const shared_value = shared_data + word_count.value * ( threadIdx.y ? threadIdx.y - 1 : blockDim.y ); + + if ( ! threadIdx.y ) { ValueInit::init( functor , shared_value ); } + + // Join previous inclusive scan value to each member + for ( size_type i = b ; i < e ; ++i ) { + size_type * const global_value = global_data + word_count.value * i ; + ValueJoin::join( functor , shared_value , global_value ); + ValueOps ::copy( functor , global_value , shared_value ); + } + } + } + + return is_last_block ; +} + +// Size in bytes required for inter block reduce or scan +template< bool DoScan , class FunctorType , class ArgTag > +inline +unsigned cuda_single_inter_block_reduce_scan_shmem( const FunctorType & functor , const unsigned BlockSize ) +{ + return ( BlockSize + 2 ) * Impl::FunctorValueTraits< FunctorType , ArgTag >::value_size( functor ); +} + +} // namespace Impl +} // namespace Kokkos + +//---------------------------------------------------------------------------- +//---------------------------------------------------------------------------- + +#endif /* #if defined( __CUDACC__ ) */ +#endif /* KOKKOS_CUDA_REDUCESCAN_HPP */ + diff --git a/lib/kokkos/core/src/Cuda/Kokkos_Cuda_Vectorization.hpp b/lib/kokkos/core/src/Cuda/Kokkos_Cuda_Vectorization.hpp new file mode 100755 index 0000000000..d6d1baa4e0 --- /dev/null +++ b/lib/kokkos/core/src/Cuda/Kokkos_Cuda_Vectorization.hpp @@ -0,0 +1,291 @@ +/* +//@HEADER +// ************************************************************************ +// +// Kokkos: Manycore Performance-Portable Multidimensional Arrays +// Copyright (2012) Sandia Corporation +// +// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +// the U.S. Government retains certain rights in this software. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// 1. Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// 3. Neither the name of the Corporation nor the names of the +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Questions? Contact H. Carter Edwards (hcedwar@sandia.gov) +// +// ************************************************************************ +//@HEADER +*/ +#ifndef KOKKOS_CUDA_VECTORIZATION_HPP +#define KOKKOS_CUDA_VECTORIZATION_HPP +#include + +namespace Kokkos { + + +// Shuffle only makes sense on >= Kepler GPUs; it doesn't work on CPUs +// or other GPUs. We provide a generic definition (which is trivial +// and doesn't do what it claims to do) because we don't actually use +// this function unless we are on a suitable GPU, with a suitable +// Scalar type. (For example, in the mat-vec, the "ThreadsPerRow" +// internal parameter depends both on the ExecutionSpace and the Scalar type, +// and it controls whether shfl_down() gets called.) +namespace Impl { + + template< typename Scalar > + struct shfl_union { + enum {n = sizeof(Scalar)/4}; + float fval[n]; + KOKKOS_INLINE_FUNCTION + Scalar value() { + return *(Scalar*) fval; + } + KOKKOS_INLINE_FUNCTION + void operator= (Scalar& value) { + float* const val_ptr = (float*) &value; + for(int i=0; i= 300) + + KOKKOS_INLINE_FUNCTION + int shfl(const int &val, const int& srcLane, const int& width ) { + return __shfl(val,srcLane,width); + } + + KOKKOS_INLINE_FUNCTION + float shfl(const float &val, const int& srcLane, const int& width ) { + return __shfl(val,srcLane,width); + } + + template + KOKKOS_INLINE_FUNCTION + Scalar shfl(const Scalar &val, const int& srcLane, const typename Impl::enable_if< (sizeof(Scalar) == 4) , int >::type& width + ) { + Scalar tmp1 = val; + float tmp = *reinterpret_cast(&tmp1); + tmp = __shfl(tmp,srcLane,width); + return *reinterpret_cast(&tmp); + } + + KOKKOS_INLINE_FUNCTION + double shfl(const double &val, const int& srcLane, const int& width) { + int lo = __double2loint(val); + int hi = __double2hiint(val); + lo = __shfl(lo,srcLane,width); + hi = __shfl(hi,srcLane,width); + return __hiloint2double(hi,lo); + } + + template + KOKKOS_INLINE_FUNCTION + Scalar shfl(const Scalar &val, const int& srcLane, const typename Impl::enable_if< (sizeof(Scalar) == 8) ,int>::type& width) { + int lo = __double2loint(*reinterpret_cast(&val)); + int hi = __double2hiint(*reinterpret_cast(&val)); + lo = __shfl(lo,srcLane,width); + hi = __shfl(hi,srcLane,width); + const double tmp = __hiloint2double(hi,lo); + return *(reinterpret_cast(&tmp)); + } + + template + KOKKOS_INLINE_FUNCTION + Scalar shfl(const Scalar &val, const int& srcLane, const typename Impl::enable_if< (sizeof(Scalar) > 8) ,int>::type& width) { + Impl::shfl_union s_val; + Impl::shfl_union r_val; + s_val = val; + + for(int i = 0; i + KOKKOS_INLINE_FUNCTION + Scalar shfl_down(const Scalar &val, const int& delta, const typename Impl::enable_if< (sizeof(Scalar) == 4) , int >::type & width) { + Scalar tmp1 = val; + float tmp = *reinterpret_cast(&tmp1); + tmp = __shfl_down(tmp,delta,width); + return *reinterpret_cast(&tmp); + } + + KOKKOS_INLINE_FUNCTION + double shfl_down(const double &val, const int& delta, const int& width) { + int lo = __double2loint(val); + int hi = __double2hiint(val); + lo = __shfl_down(lo,delta,width); + hi = __shfl_down(hi,delta,width); + return __hiloint2double(hi,lo); + } + + template + KOKKOS_INLINE_FUNCTION + Scalar shfl_down(const Scalar &val, const int& delta, const typename Impl::enable_if< (sizeof(Scalar) == 8) , int >::type & width) { + int lo = __double2loint(*reinterpret_cast(&val)); + int hi = __double2hiint(*reinterpret_cast(&val)); + lo = __shfl_down(lo,delta,width); + hi = __shfl_down(hi,delta,width); + const double tmp = __hiloint2double(hi,lo); + return *(reinterpret_cast(&tmp)); + } + + template + KOKKOS_INLINE_FUNCTION + Scalar shfl_down(const Scalar &val, const int& delta, const typename Impl::enable_if< (sizeof(Scalar) > 8) , int >::type & width) { + Impl::shfl_union s_val; + Impl::shfl_union r_val; + s_val = val; + + for(int i = 0; i + KOKKOS_INLINE_FUNCTION + Scalar shfl_up(const Scalar &val, const int& delta, const typename Impl::enable_if< (sizeof(Scalar) == 4) , int >::type & width) { + Scalar tmp1 = val; + float tmp = *reinterpret_cast(&tmp1); + tmp = __shfl_up(tmp,delta,width); + return *reinterpret_cast(&tmp); + } + + KOKKOS_INLINE_FUNCTION + double shfl_up(const double &val, const int& delta, const int& width ) { + int lo = __double2loint(val); + int hi = __double2hiint(val); + lo = __shfl_up(lo,delta,width); + hi = __shfl_up(hi,delta,width); + return __hiloint2double(hi,lo); + } + + template + KOKKOS_INLINE_FUNCTION + Scalar shfl_up(const Scalar &val, const int& delta, const typename Impl::enable_if< (sizeof(Scalar) == 8) , int >::type & width) { + int lo = __double2loint(*reinterpret_cast(&val)); + int hi = __double2hiint(*reinterpret_cast(&val)); + lo = __shfl_up(lo,delta,width); + hi = __shfl_up(hi,delta,width); + const double tmp = __hiloint2double(hi,lo); + return *(reinterpret_cast(&tmp)); + } + + template + KOKKOS_INLINE_FUNCTION + Scalar shfl_up(const Scalar &val, const int& delta, const typename Impl::enable_if< (sizeof(Scalar) > 8) , int >::type & width) { + Impl::shfl_union s_val; + Impl::shfl_union r_val; + s_val = val; + + for(int i = 0; i + KOKKOS_INLINE_FUNCTION + Scalar shfl(const Scalar &val, const int& srcLane, const int& width) { + if(width > 1) Kokkos::abort("Error: calling shfl from a device with CC<3.0."); + return val; + } + + template + KOKKOS_INLINE_FUNCTION + Scalar shfl_down(const Scalar &val, const int& delta, const int& width) { + if(width > 1) Kokkos::abort("Error: calling shfl_down from a device with CC<3.0."); + return val; + } + + template + KOKKOS_INLINE_FUNCTION + Scalar shfl_up(const Scalar &val, const int& delta, const int& width) { + if(width > 1) Kokkos::abort("Error: calling shfl_down from a device with CC<3.0."); + return val; + } + #endif +#else + template + inline + Scalar shfl(const Scalar &val, const int& srcLane, const int& width) { + if(width > 1) Kokkos::abort("Error: calling shfl from a device with CC<3.0."); + return val; + } + + template + inline + Scalar shfl_down(const Scalar &val, const int& delta, const int& width) { + if(width > 1) Kokkos::abort("Error: calling shfl_down from a device with CC<3.0."); + return val; + } + + template + inline + Scalar shfl_up(const Scalar &val, const int& delta, const int& width) { + if(width > 1) Kokkos::abort("Error: calling shfl_down from a device with CC<3.0."); + return val; + } +#endif + + + +} + +#endif diff --git a/lib/kokkos/core/src/Cuda/Kokkos_Cuda_View.hpp b/lib/kokkos/core/src/Cuda/Kokkos_Cuda_View.hpp new file mode 100755 index 0000000000..67c7214f89 --- /dev/null +++ b/lib/kokkos/core/src/Cuda/Kokkos_Cuda_View.hpp @@ -0,0 +1,299 @@ +/* +//@HEADER +// ************************************************************************ +// +// Kokkos: Manycore Performance-Portable Multidimensional Arrays +// Copyright (2012) Sandia Corporation +// +// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +// the U.S. Government retains certain rights in this software. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// 1. Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// 3. Neither the name of the Corporation nor the names of the +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Questions? Contact H. Carter Edwards (hcedwar@sandia.gov) +// +// ************************************************************************ +//@HEADER +*/ + +#ifndef KOKKOS_CUDA_VIEW_HPP +#define KOKKOS_CUDA_VIEW_HPP + +#include + +#include +#include +#include +#include + +//---------------------------------------------------------------------------- +//---------------------------------------------------------------------------- + +namespace Kokkos { +namespace Impl { + +template<> +struct AssertShapeBoundsAbort< CudaSpace > +{ + KOKKOS_INLINE_FUNCTION + static void apply( const size_t /* rank */ , + const size_t /* n0 */ , const size_t /* n1 */ , + const size_t /* n2 */ , const size_t /* n3 */ , + const size_t /* n4 */ , const size_t /* n5 */ , + const size_t /* n6 */ , const size_t /* n7 */ , + + const size_t /* arg_rank */ , + const size_t /* i0 */ , const size_t /* i1 */ , + const size_t /* i2 */ , const size_t /* i3 */ , + const size_t /* i4 */ , const size_t /* i5 */ , + const size_t /* i6 */ , const size_t /* i7 */ ) + { + Kokkos::abort("Kokkos::View array bounds violation"); + } +}; + +} +} + +//---------------------------------------------------------------------------- +//---------------------------------------------------------------------------- + +namespace Kokkos { +namespace Impl { + +// Cuda 5.0 defines 'cudaTextureObject_t' +// to be an 'unsigned long long'. This chould change with +// future version of Cuda and this typedef would have to +// change accordingly. + +#if defined( CUDA_VERSION ) && ( 5000 <= CUDA_VERSION ) + +typedef enable_if< + sizeof(::cudaTextureObject_t) == sizeof(const void *) , + ::cudaTextureObject_t >::type cuda_texture_object_type ; + +#else + +typedef const void * cuda_texture_object_type ; + +#endif + +//---------------------------------------------------------------------------- +// Cuda Texture fetches can be performed for 4, 8 and 16 byte objects (int,int2,int4) +// Via reinterpret_case this can be used to support all scalar types of those sizes. +// Any other scalar type falls back to either normal reads out of global memory, +// or using the __ldg intrinsic on Kepler GPUs or newer (Compute Capability >= 3.0) + +template< typename ValueType + , class MemorySpace + , class AliasType = + typename Kokkos::Impl::if_c< ( sizeof(ValueType) == 4 ) , int , + typename Kokkos::Impl::if_c< ( sizeof(ValueType) == 8 ) , int2 , + typename Kokkos::Impl::if_c< ( sizeof(ValueType) == 16 ) , int4 , void + >::type + >::type + >::type + > +class CudaTextureFetch { +private: + + cuda_texture_object_type m_obj ; + const ValueType * m_alloc_ptr ; + int m_offset ; + +public: + + KOKKOS_INLINE_FUNCTION + CudaTextureFetch() : m_obj( 0 ) , m_alloc_ptr(0) , m_offset(0) {} + + KOKKOS_INLINE_FUNCTION + ~CudaTextureFetch() {} + + KOKKOS_INLINE_FUNCTION + CudaTextureFetch( const CudaTextureFetch & rhs ) + : m_obj( rhs.m_obj ) + , m_alloc_ptr( rhs.m_alloc_ptr ) + , m_offset( rhs.m_offset ) + {} + + KOKKOS_INLINE_FUNCTION + CudaTextureFetch & operator = ( const CudaTextureFetch & rhs ) + { + m_obj = rhs.m_obj ; + m_alloc_ptr = rhs.m_alloc_ptr ; + m_offset = rhs.m_offset ; + return *this ; + } + + + KOKKOS_INLINE_FUNCTION explicit + CudaTextureFetch( const ValueType * const arg_ptr ) + : m_obj( 0 ) , m_alloc_ptr(0) , m_offset(0) + { +#if defined( __CUDACC__ ) && ! defined( __CUDA_ARCH__ ) + MemorySpace::texture_object_attach( arg_ptr + , sizeof(ValueType) + , cudaCreateChannelDesc< AliasType >() + , & m_obj + , reinterpret_cast( & m_alloc_ptr ) + , & m_offset + ); +#endif + } + + KOKKOS_INLINE_FUNCTION + CudaTextureFetch & operator = ( const ValueType * arg_ptr ) + { +#if defined( __CUDACC__ ) && ! defined( __CUDA_ARCH__ ) + MemorySpace::texture_object_attach( arg_ptr + , sizeof(ValueType) + , cudaCreateChannelDesc< AliasType >() + , & m_obj + , reinterpret_cast( & m_alloc_ptr ) + , & m_offset + ); +#endif + return *this ; + } + + + KOKKOS_INLINE_FUNCTION + operator const ValueType * () const { return m_alloc_ptr + m_offset ; } + + + template< typename iType > + KOKKOS_INLINE_FUNCTION + ValueType operator[]( const iType & i ) const + { +#if defined( __CUDA_ARCH__ ) && ( 300 <= __CUDA_ARCH__ ) +#if defined( KOKKOS_USE_LDG_INTRINSIC ) + // Enable the usage of the _ldg intrinsic even in cases where texture fetches work + // Currently texture fetches are faster, but that might change in the future + return _ldg( & m_alloc_ptr[i+m_offset] ); +#else /* ! defined( KOKKOS_USE_LDG_INTRINSIC ) */ + AliasType v = tex1Dfetch( m_obj , i + m_offset ); + + return *(reinterpret_cast (&v)); +#endif /* ! defined( KOKKOS_USE_LDG_INTRINSIC ) */ +#else /* ! defined( __CUDA_ARCH__ ) && ( 300 <= __CUDA_ARCH__ ) */ + return m_alloc_ptr[ i + m_offset ]; +#endif + } +}; + +template< typename ValueType > +class CudaTextureFetch< const ValueType, void > +{ +private: + const ValueType * m_ptr ; +public: + + KOKKOS_INLINE_FUNCTION + CudaTextureFetch() : m_ptr(0) {}; + + KOKKOS_INLINE_FUNCTION + ~CudaTextureFetch() { + } + + KOKKOS_INLINE_FUNCTION + CudaTextureFetch( const CudaTextureFetch & rhs ) : m_ptr(rhs.m_ptr) {} + + KOKKOS_INLINE_FUNCTION + CudaTextureFetch & operator = ( const CudaTextureFetch & rhs ) { + m_ptr = rhs.m_ptr; + return *this ; + } + + explicit KOKKOS_INLINE_FUNCTION + CudaTextureFetch( ValueType * const base_view_ptr ) { + m_ptr = base_view_ptr; + } + + KOKKOS_INLINE_FUNCTION + CudaTextureFetch & operator = (const ValueType* base_view_ptr) { + m_ptr = base_view_ptr; + return *this; + } + + + KOKKOS_INLINE_FUNCTION + operator const ValueType * () const { return m_ptr ; } + + + template< typename iType > + KOKKOS_INLINE_FUNCTION + ValueType operator[]( const iType & i ) const + { + #if defined( __CUDA_ARCH__ ) && ( 300 <= __CUDA_ARCH__ ) + return _ldg(&m_ptr[i]); + #else + return m_ptr[ i ]; + #endif + } +}; + +} // namespace Impl +} // namespace Kokkos + + +//---------------------------------------------------------------------------- +//---------------------------------------------------------------------------- + +namespace Kokkos { +namespace Impl { + +/** \brief Replace Default ViewDataHandle with Cuda texture fetch specialization + * if 'const' value type, CudaSpace and random access. + */ +template< class ViewTraits > +class ViewDataHandle< ViewTraits , + typename enable_if< ( is_same< typename ViewTraits::memory_space,CudaSpace>::value || + is_same< typename ViewTraits::memory_space,CudaUVMSpace>::value ) + && + is_same::value + && + ViewTraits::memory_traits::RandomAccess + >::type > +{ +public: + enum { ReturnTypeIsReference = false }; + + typedef Impl::CudaTextureFetch< typename ViewTraits::value_type + , typename ViewTraits::memory_space > handle_type; + + typedef typename ViewTraits::value_type return_type; +}; + +} +} + +//---------------------------------------------------------------------------- +//---------------------------------------------------------------------------- + +#endif /* #ifndef KOKKOS_CUDA_VIEW_HPP */ + diff --git a/lib/kokkos/core/src/Cuda/Kokkos_Cuda_abort.hpp b/lib/kokkos/core/src/Cuda/Kokkos_Cuda_abort.hpp new file mode 100755 index 0000000000..bd85259ce0 --- /dev/null +++ b/lib/kokkos/core/src/Cuda/Kokkos_Cuda_abort.hpp @@ -0,0 +1,117 @@ +/* +//@HEADER +// ************************************************************************ +// +// Kokkos: Manycore Performance-Portable Multidimensional Arrays +// Copyright (2012) Sandia Corporation +// +// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +// the U.S. Government retains certain rights in this software. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// 1. Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// 3. Neither the name of the Corporation nor the names of the +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Questions? Contact H. Carter Edwards (hcedwar@sandia.gov) +// +// ************************************************************************ +//@HEADER +*/ + +#ifndef KOKKOS_CUDA_ABORT_HPP +#define KOKKOS_CUDA_ABORT_HPP + +//---------------------------------------------------------------------------- +//---------------------------------------------------------------------------- + +#if defined( __CUDACC__ ) && defined( __CUDA_ARCH__ ) + +#include + +#if ! defined( CUDA_VERSION ) || ( CUDA_VERSION < 4010 ) +#error "Cuda version 4.1 or greater required" +#endif + +#if ( __CUDA_ARCH__ < 200 ) +#error "Cuda device capability 2.0 or greater required" +#endif + +extern "C" { +/* Cuda runtime function, declared in + * Requires capability 2.x or better. + */ +extern __device__ void __assertfail( + const void *message, + const void *file, + unsigned int line, + const void *function, + size_t charsize); +} + +namespace Kokkos { +namespace Impl { + +__device__ inline +void cuda_abort( const char * const message ) +{ + const char empty[] = "" ; + + __assertfail( (const void *) message , + (const void *) empty , + (unsigned int) 0 , + (const void *) empty , + sizeof(char) ); +} + +} // namespace Impl +} // namespace Kokkos + +#else + +namespace Kokkos { +namespace Impl { +KOKKOS_INLINE_FUNCTION +void cuda_abort( const char * const ) {} +} +} + +#endif /* #if defined( __CUDACC__ ) && defined( __CUDA_ARCH__ ) */ + +//---------------------------------------------------------------------------- +//---------------------------------------------------------------------------- + +#if defined( KOKKOS_ACTIVE_EXECUTION_MEMORY_SPACE_CUDA ) +namespace Kokkos { +__device__ inline +void abort( const char * const message ) { Kokkos::Impl::cuda_abort(message); } +} +#endif /* defined( KOKKOS_ACTIVE_EXECUTION_MEMORY_SPACE_CUDA ) */ + +//---------------------------------------------------------------------------- +//---------------------------------------------------------------------------- + +#endif /* #ifndef KOKKOS_CUDA_ABORT_HPP */ + diff --git a/lib/kokkos/core/src/Kokkos_Atomic.hpp b/lib/kokkos/core/src/Kokkos_Atomic.hpp new file mode 100755 index 0000000000..856c740ea7 --- /dev/null +++ b/lib/kokkos/core/src/Kokkos_Atomic.hpp @@ -0,0 +1,236 @@ +/* +//@HEADER +// ************************************************************************ +// +// Kokkos: Manycore Performance-Portable Multidimensional Arrays +// Copyright (2012) Sandia Corporation +// +// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +// the U.S. Government retains certain rights in this software. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// 1. Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// 3. Neither the name of the Corporation nor the names of the +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Questions? Contact H. Carter Edwards (hcedwar@sandia.gov) +// +// ************************************************************************ +//@HEADER +*/ + +/// \file Kokkos_Atomic.hpp +/// \brief Atomic functions +/// +/// This header file defines prototypes for the following atomic functions: +/// - exchange +/// - compare and exchange +/// - add +/// +/// Supported types include: +/// - signed and unsigned 4 and 8 byte integers +/// - float +/// - double +/// +/// They are implemented through GCC compatible intrinsics, OpenMP +/// directives and native CUDA intrinsics. +/// +/// Including this header file requires one of the following +/// compilers: +/// - NVCC (for CUDA device code only) +/// - GCC (for host code only) +/// - Intel (for host code only) +/// - A compiler that supports OpenMP 3.1 (for host code only) + +#ifndef KOKKOS_ATOMIC_HPP +#define KOKKOS_ATOMIC_HPP + +#include +#include + +//---------------------------------------------------------------------------- + +#if defined( __CUDA_ARCH__ ) + +// Compiling NVIDIA device code, must use Cuda atomics: + +#define KOKKOS_ATOMICS_USE_CUDA + +#elif ! defined( KOKKOS_ATOMICS_USE_GCC ) && \ + ! defined( KOKKOS_ATOMICS_USE_INTEL ) && \ + ! defined( KOKKOS_ATOMICS_USE_OMP31 ) + +// Compiling for non-Cuda atomic implementation has not been pre-selected. +// Choose the best implementation for the detected compiler. +// Preference: GCC, INTEL, OMP31 + +#if defined( KOKKOS_COMPILER_GNU ) || \ + defined( KOKKOS_COMPILER_CLANG ) + +#define KOKKOS_ATOMICS_USE_GCC + +#elif defined( KOKKOS_COMPILER_INTEL ) || \ + defined( KOKKOS_COMPILER_CRAYC ) + +#define KOKKOS_ATOMICS_USE_INTEL + +#elif defined( _OPENMP ) && ( 201107 <= _OPENMP ) + +#define KOKKOS_ATOMICS_USE_OMP31 + +#else + +#error "KOKKOS_ATOMICS_USE : Unsupported compiler" + +#endif + +#endif /* Not pre-selected atomic implementation */ + +//---------------------------------------------------------------------------- + +namespace Kokkos { +template +KOKKOS_INLINE_FUNCTION +void atomic_add(volatile T * const dest, const T src); + +// Atomic increment +template +KOKKOS_INLINE_FUNCTION +void atomic_increment(volatile T* a); + +template +KOKKOS_INLINE_FUNCTION +void atomic_decrement(volatile T* a); +} + + +#include + +namespace Kokkos { + + +inline +const char * atomic_query_version() +{ +#if defined( KOKKOS_ATOMICS_USE_CUDA ) + return "KOKKOS_ATOMICS_USE_CUDA" ; +#elif defined( KOKKOS_ATOMICS_USE_GCC ) + return "KOKKOS_ATOMICS_USE_GCC" ; +#elif defined( KOKKOS_ATOMICS_USE_INTEL ) + return "KOKKOS_ATOMICS_USE_INTEL" ; +#elif defined( KOKKOS_ATOMICS_USE_OMP31 ) + return "KOKKOS_ATOMICS_USE_OMP31" ; +#endif +} + +} // namespace Kokkos + +//#include "impl/Kokkos_Atomic_Assembly_X86.hpp" + +//---------------------------------------------------------------------------- +// Atomic exchange +// +// template< typename T > +// T atomic_exchange( volatile T* const dest , const T val ) +// { T tmp = *dest ; *dest = val ; return tmp ; } + +#include "impl/Kokkos_Atomic_Exchange.hpp" + +//---------------------------------------------------------------------------- +// Atomic compare-and-exchange +// +// template +// bool atomic_compare_exchange_strong(volatile T* const dest, const T compare, const T val) +// { bool equal = compare == *dest ; if ( equal ) { *dest = val ; } return equal ; } + +#include "impl/Kokkos_Atomic_Compare_Exchange_Strong.hpp" + +//---------------------------------------------------------------------------- +// Atomic fetch and add +// +// template +// T atomic_fetch_add(volatile T* const dest, const T val) +// { T tmp = *dest ; *dest += val ; return tmp ; } + +#include "impl/Kokkos_Atomic_Fetch_Add.hpp" + +//---------------------------------------------------------------------------- +// Atomic fetch and or +// +// template +// T atomic_fetch_or(volatile T* const dest, const T val) +// { T tmp = *dest ; *dest = tmp | val ; return tmp ; } + +#include "impl/Kokkos_Atomic_Fetch_Or.hpp" + +//---------------------------------------------------------------------------- +// Atomic fetch and and +// +// template +// T atomic_fetch_and(volatile T* const dest, const T val) +// { T tmp = *dest ; *dest = tmp & val ; return tmp ; } + +#include "impl/Kokkos_Atomic_Fetch_And.hpp" + +//---------------------------------------------------------------------------- +// Memory fence +// +// All loads and stores from this thread will be globally consistent before continuing +// +// void memory_fence() {...}; +#include "impl/Kokkos_Memory_Fence.hpp" + +//---------------------------------------------------------------------------- +// Provide volatile_load and safe_load +// +// T volatile_load(T const volatile * const ptr); +// +// T const& safe_load(T const * const ptr); +// XEON PHI +// T safe_load(T const * const ptr + +#include "impl/Kokkos_Volatile_Load.hpp" + +#include "impl/Kokkos_Atomic_Generic.hpp" + +//---------------------------------------------------------------------------- +// This atomic-style macro should be an inlined function, not a macro + +#if defined( KOKKOS_COMPILER_GNU ) + + #define KOKKOS_NONTEMPORAL_PREFETCH_LOAD(addr) __builtin_prefetch(addr,0,0) + #define KOKKOS_NONTEMPORAL_PREFETCH_STORE(addr) __builtin_prefetch(addr,1,0) + +#else + + #define KOKKOS_NONTEMPORAL_PREFETCH_LOAD(addr) ((void)0) + #define KOKKOS_NONTEMPORAL_PREFETCH_STORE(addr) ((void)0) + +#endif + +//---------------------------------------------------------------------------- + +#endif /* KOKKOS_ATOMIC_HPP */ + diff --git a/lib/kokkos/core/src/Kokkos_Core.hpp b/lib/kokkos/core/src/Kokkos_Core.hpp new file mode 100755 index 0000000000..8f5f34bfd3 --- /dev/null +++ b/lib/kokkos/core/src/Kokkos_Core.hpp @@ -0,0 +1,106 @@ +/* +//@HEADER +// ************************************************************************ +// +// Kokkos +// Manycore Performance-Portable Multidimensional Arrays +// +// Copyright (2012) Sandia Corporation +// +// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +// the U.S. Government retains certain rights in this software. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// 1. Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// 3. Neither the name of the Corporation nor the names of the +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Questions? Contact H. Carter Edwards (hcedwar@sandia.gov) +// +// ************************************************************************ +//@HEADER +*/ + +#ifndef KOKKOS_CORE_HPP +#define KOKKOS_CORE_HPP + +//---------------------------------------------------------------------------- +// Include the execution space header files for the enabled execution spaces. + +#include + +#if defined( KOKKOS_HAVE_CUDA ) +#include +#endif + +#if defined( KOKKOS_HAVE_OPENMP ) +#include +#endif + +#if defined( KOKKOS_HAVE_SERIAL ) +#include +#endif + +#if defined( KOKKOS_HAVE_PTHREAD ) +#include +#endif + +#include +#include +#include +#include +#include + +//---------------------------------------------------------------------------- + +namespace Kokkos { + +struct InitArguments { + int num_threads; + int num_numa; + int device_id; + + InitArguments() { + num_threads = -1; + num_numa = -1; + device_id = -1; + } +}; + +void initialize(int& narg, char* arg[]); + +void initialize(const InitArguments& args = InitArguments()); + +/** \brief Finalize the spaces that were initialized via Kokkos::initialize */ +void finalize(); + +/** \brief Finalize all known execution spaces */ +void finalize_all(); + +void fence(); + +} + +#endif diff --git a/lib/kokkos/core/src/Kokkos_Core_fwd.hpp b/lib/kokkos/core/src/Kokkos_Core_fwd.hpp new file mode 100755 index 0000000000..2661d315a3 --- /dev/null +++ b/lib/kokkos/core/src/Kokkos_Core_fwd.hpp @@ -0,0 +1,150 @@ +/* +//@HEADER +// ************************************************************************ +// +// Kokkos +// Manycore Performance-Portable Multidimensional Arrays +// +// Copyright (2012) Sandia Corporation +// +// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +// the U.S. Government retains certain rights in this software. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// 1. Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// 3. Neither the name of the Corporation nor the names of the +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Questions? Contact H. Carter Edwards (hcedwar@sandia.gov) +// +// ************************************************************************ +//@HEADER +*/ + +#ifndef KOKKOS_CORE_FWD_HPP +#define KOKKOS_CORE_FWD_HPP + +//---------------------------------------------------------------------------- +// Kokkos_Macros.hpp does introspection on configuration options +// and compiler environment then sets a collection of #define macros. + +#include + +//---------------------------------------------------------------------------- +//---------------------------------------------------------------------------- +// Forward declarations for class inter-relationships + +namespace Kokkos { + +class HostSpace ; ///< Memory space for main process and CPU execution spaces + +#if defined( KOKKOS_HAVE_SERIAL ) +class Serial ; ///< Execution space main process on CPU +#endif // defined( KOKKOS_HAVE_SERIAL ) + +#if defined( KOKKOS_HAVE_PTHREAD ) +class Threads ; ///< Execution space with pthreads back-end +#endif + +#if defined( KOKKOS_HAVE_OPENMP ) +class OpenMP ; ///< OpenMP execution space +#endif + +#if defined( KOKKOS_HAVE_CUDA ) +class CudaSpace ; ///< Memory space on Cuda GPU +class CudaUVMSpace ; ///< Memory space on Cuda GPU with UVM +class CudaHostPinnedSpace ; ///< Memory space on Host accessible to Cuda GPU +class Cuda ; ///< Execution space for Cuda GPU +#endif + +} // namespace Kokkos + +//---------------------------------------------------------------------------- +//---------------------------------------------------------------------------- +// Set the default execution space. + +/// Define Kokkos::DefaultExecutionSpace as per configuration option +/// or chosen from the enabled execution spaces in the following order: +/// Kokkos::Cuda, Kokkos::OpenMP, Kokkos::Threads, Kokkos::Serial + +namespace Kokkos { + +#if defined ( KOKKOS_HAVE_DEFAULT_DEVICE_TYPE_CUDA ) + typedef Kokkos::Cuda DefaultExecutionSpace ; +#elif defined ( KOKKOS_HAVE_DEFAULT_DEVICE_TYPE_OPENMP ) + typedef OpenMP DefaultExecutionSpace ; +#elif defined ( KOKKOS_HAVE_DEFAULT_DEVICE_TYPE_THREADS ) + typedef Threads DefaultExecutionSpace ; +#elif defined ( KOKKOS_HAVE_DEFAULT_DEVICE_TYPE_SERIAL ) + typedef Serial DefaultExecutionSpace ; +#else +# error "At least one of the following execution spaces must be defined in order to use Kokkos: Kokkos::Cuda, Kokkos::OpenMP, Kokkos::Serial, or Kokkos::Threads." +#endif + +} // namespace Kokkos + +//---------------------------------------------------------------------------- +//---------------------------------------------------------------------------- +// Detect the active execution space and define its memory space. +// This is used to verify whether a running kernel can access +// a given memory space. + +namespace Kokkos { +namespace Impl { + +#if defined( KOKKOS_ACTIVE_EXECUTION_MEMORY_SPACE_CUDA ) && defined (KOKKOS_HAVE_CUDA) +typedef Kokkos::CudaSpace ActiveExecutionMemorySpace ; +#elif defined( KOKKOS_ACTIVE_EXECUTION_MEMORY_SPACE_HOST ) +typedef Kokkos::HostSpace ActiveExecutionMemorySpace ; +#else +typedef void ActiveExecutionMemorySpace ; +#endif + +template< class ActiveSpace , class MemorySpace > +struct VerifyExecutionCanAccessMemorySpace { + enum {value = 0}; +}; + +template< class Space > +struct VerifyExecutionCanAccessMemorySpace< Space , Space > +{ + enum {value = 1}; + KOKKOS_INLINE_FUNCTION static void verify(void) {} + KOKKOS_INLINE_FUNCTION static void verify(const void *) {} +}; + +} // namespace Impl +} // namespace Kokkos + +#define KOKKOS_RESTRICT_EXECUTION_TO_DATA( DATA_SPACE , DATA_PTR ) \ + Kokkos::Impl::VerifyExecutionCanAccessMemorySpace< \ + Kokkos::Impl::ActiveExecutionMemorySpace , DATA_SPACE >::verify( DATA_PTR ) + +#define KOKKOS_RESTRICT_EXECUTION_TO_( DATA_SPACE ) \ + Kokkos::Impl::VerifyExecutionCanAccessMemorySpace< \ + Kokkos::Impl::ActiveExecutionMemorySpace , DATA_SPACE >::verify() + +#endif /* #ifndef KOKKOS_CORE_FWD_HPP */ + diff --git a/lib/kokkos/core/src/Kokkos_CrsArray.hpp b/lib/kokkos/core/src/Kokkos_CrsArray.hpp new file mode 100755 index 0000000000..53ab15b218 --- /dev/null +++ b/lib/kokkos/core/src/Kokkos_CrsArray.hpp @@ -0,0 +1,171 @@ +/* +//@HEADER +// ************************************************************************ +// +// Kokkos +// Manycore Performance-Portable Multidimensional Arrays +// +// Copyright (2012) Sandia Corporation +// +// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +// the U.S. Government retains certain rights in this software. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// 1. Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// 3. Neither the name of the Corporation nor the names of the +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Questions? Contact H. Carter Edwards (hcedwar@sandia.gov) +// +// ************************************************************************ +//@HEADER +*/ + +#ifndef KOKKOS_CRSARRAY_HPP +#define KOKKOS_CRSARRAY_HPP + +#include +#include + +#include + +namespace Kokkos { + +/// \class CrsArray +/// \brief Compressed row storage array. +/// +/// \tparam DataType The type of stored entries. If a CrsArray is +/// used as the graph of a sparse matrix, then this is usually an +/// integer type, the type of the column indices in the sparse +/// matrix. +/// +/// \tparam Arg1Type The second template parameter, corresponding +/// either to the Space type (if there are no more template +/// parameters) or to the Layout type (if there is at least one more +/// template parameter). +/// +/// \tparam Arg2Type The third template parameter, which if provided +/// corresponds to the Space type. +/// +/// \tparam SizeType The type of row offsets. Usually the default +/// parameter suffices. However, setting a nondefault value is +/// necessary in some cases, for example, if you want to have a +/// sparse matrices with dimensions (and therefore column indices) +/// that fit in \c int, but want to store more than INT_MAX +/// entries in the sparse matrix. +/// +/// A row has a range of entries: +///
    +///
  • row_map[i0] <= entry < row_map[i0+1]
  • +///
  • 0 <= i1 < row_map[i0+1] - row_map[i0]
  • +///
  • entries( entry , i2 , i3 , ... );
  • +///
  • entries( row_map[i0] + i1 , i2 , i3 , ... );
  • +///
+template< class DataType, + class Arg1Type, + class Arg2Type = void, + typename SizeType = typename ViewTraits::size_type> +class CrsArray { +private: + typedef ViewTraits traits ; + +public: + typedef DataType data_type; + typedef typename traits::array_layout array_layout; + typedef typename traits::execution_space execution_space ; + typedef typename traits::memory_space memory_space ; + typedef SizeType size_type; + + typedef CrsArray< DataType , Arg1Type , Arg2Type , SizeType > crsarray_type; + typedef CrsArray< DataType , array_layout , typename traits::host_mirror_space , SizeType > HostMirror; + typedef View< const size_type* , array_layout, execution_space > row_map_type; + typedef View< DataType* , array_layout, execution_space > entries_type; + + entries_type entries; + row_map_type row_map; + + //! Construct an empty view. + CrsArray () : entries(), row_map() {} + + //! Copy constructor (shallow copy). + CrsArray (const CrsArray& rhs) : entries (rhs.entries), row_map (rhs.row_map) + {} + + /** \brief Assign to a view of the rhs array. + * If the old view is the last view + * then allocated memory is deallocated. + */ + CrsArray& operator= (const CrsArray& rhs) { + entries = rhs.entries; + row_map = rhs.row_map; + return *this; + } + + /** \brief Destroy this view of the array. + * If the last view then allocated memory is deallocated. + */ + ~CrsArray() {} +}; + +//---------------------------------------------------------------------------- + +template< class CrsArrayType , class InputSizeType > +typename CrsArrayType::crsarray_type +create_crsarray( const std::string & label , + const std::vector< InputSizeType > & input ); + +template< class CrsArrayType , class InputSizeType > +typename CrsArrayType::crsarray_type +create_crsarray( const std::string & label , + const std::vector< std::vector< InputSizeType > > & input ); + +//---------------------------------------------------------------------------- + +template< class DataType , + class Arg1Type , + class Arg2Type , + typename SizeType > +typename CrsArray< DataType , Arg1Type , Arg2Type , SizeType >::HostMirror +create_mirror_view( const CrsArray & input ); + +template< class DataType , + class Arg1Type , + class Arg2Type , + typename SizeType > +typename CrsArray< DataType , Arg1Type , Arg2Type , SizeType >::HostMirror +create_mirror( const CrsArray & input ); + +} // namespace Kokkos + +//---------------------------------------------------------------------------- +//---------------------------------------------------------------------------- + +#include + +//---------------------------------------------------------------------------- +//---------------------------------------------------------------------------- + +#endif /* #ifndef KOKKOS_CRSARRAY_HPP */ + diff --git a/lib/kokkos/core/src/Kokkos_Cuda.hpp b/lib/kokkos/core/src/Kokkos_Cuda.hpp new file mode 100755 index 0000000000..e3325a9758 --- /dev/null +++ b/lib/kokkos/core/src/Kokkos_Cuda.hpp @@ -0,0 +1,263 @@ +/* +//@HEADER +// ************************************************************************ +// +// Kokkos +// Manycore Performance-Portable Multidimensional Arrays +// +// Copyright (2012) Sandia Corporation +// +// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +// the U.S. Government retains certain rights in this software. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// 1. Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// 3. Neither the name of the Corporation nor the names of the +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Questions? Contact H. Carter Edwards (hcedwar@sandia.gov) +// +// ************************************************************************ +//@HEADER +*/ + +#ifndef KOKKOS_CUDA_HPP +#define KOKKOS_CUDA_HPP + +#include + +// If CUDA execution space is enabled then use this header file. + +#if defined( KOKKOS_HAVE_CUDA ) + +#include +#include + +#include + +#include +#include +#include +#include +#include + +/*--------------------------------------------------------------------------*/ + +namespace Kokkos { +namespace Impl { +class CudaExec ; +} // namespace Impl +} // namespace Kokkos + +/*--------------------------------------------------------------------------*/ + +namespace Kokkos { + +/// \class Cuda +/// \brief Kokkos Execution Space that uses CUDA to run on GPUs. +/// +/// An "execution space" represents a parallel execution model. It tells Kokkos +/// how to parallelize the execution of kernels in a parallel_for or +/// parallel_reduce. For example, the Threads execution space uses Pthreads or +/// C++11 threads on a CPU, the OpenMP execution space uses the OpenMP language +/// extensions, and the Serial execution space executes "parallel" kernels +/// sequentially. The Cuda execution space uses NVIDIA's CUDA programming +/// model to execute kernels in parallel on GPUs. +class Cuda { +public: + //! \name Type declarations that all Kokkos execution spaces must provide. + //@{ + + //! Tag this class as a kokkos execution space + typedef Cuda execution_space ; + +#if defined( KOKKOS_USE_CUDA_UVM ) + //! This execution space's preferred memory space. + typedef CudaUVMSpace memory_space ; +#else + //! This execution space's preferred memory space. + typedef CudaSpace memory_space ; +#endif + + //! The size_type best suited for this execution space. + typedef memory_space::size_type size_type ; + + //! This execution space's preferred array layout. + typedef LayoutLeft array_layout ; + + //! For backward compatibility + typedef Cuda device_type ; + //! + typedef ScratchMemorySpace< Cuda > scratch_memory_space ; + + //@} + //-------------------------------------------------- + //! \name Functions that all Kokkos devices must implement. + //@{ + + /// \brief True if and only if this method is being called in a + /// thread-parallel function. + KOKKOS_INLINE_FUNCTION static int in_parallel() { +#if defined( __CUDA_ARCH__ ) + return true; +#else + return false; +#endif + } + + /** \brief Set the device in a "sleep" state. + * + * This function sets the device in a "sleep" state in which it is + * not ready for work. This may consume less resources than if the + * device were in an "awake" state, but it may also take time to + * bring the device from a sleep state to be ready for work. + * + * \return True if the device is in the "sleep" state, else false if + * the device is actively working and could not enter the "sleep" + * state. + */ + static bool sleep(); + + /// \brief Wake the device from the 'sleep' state so it is ready for work. + /// + /// \return True if the device is in the "ready" state, else "false" + /// if the device is actively working (which also means that it's + /// awake). + static bool wake(); + + /// \brief Wait until all dispatched functors complete. + /// + /// The parallel_for or parallel_reduce dispatch of a functor may + /// return asynchronously, before the functor completes. This + /// method does not return until all dispatched functors on this + /// device have completed. + static void fence(); + + //! Free any resources being consumed by the device. + static void finalize(); + + //! Has been initialized + static int is_initialized(); + + //! Print configuration information to the given output stream. + static void print_configuration( std::ostream & , const bool detail = false ); + + //@} + //-------------------------------------------------- + //! \name Cuda space instances + + ~Cuda() {} + Cuda(); + explicit Cuda( const int instance_id ); + +#if defined( KOKKOS_HAVE_CXX11 ) + Cuda & operator = ( const Cuda & ) = delete ; +#else +private: + Cuda & operator = ( const Cuda & ); +public: +#endif + + //-------------------------------------------------------------------------- + //! \name Device-specific functions + //@{ + + struct SelectDevice { + int cuda_device_id ; + SelectDevice() : cuda_device_id(0) {} + explicit SelectDevice( int id ) : cuda_device_id( id ) {} + }; + + //! Initialize, telling the CUDA run-time library which device to use. + static void initialize( const SelectDevice = SelectDevice() + , const size_t num_instances = 1 ); + + /// \brief Cuda device architecture of the selected device. + /// + /// This matches the __CUDA_ARCH__ specification. + static size_type device_arch(); + + //! Query device count. + static size_type detect_device_count(); + + /** \brief Detect the available devices and their architecture + * as defined by the __CUDA_ARCH__ specification. + */ + static std::vector detect_device_arch(); + + //@} + //-------------------------------------------------------------------------- + + const cudaStream_t m_stream ; + const int m_device ; +}; + +} // namespace Kokkos + +/*--------------------------------------------------------------------------*/ +/*--------------------------------------------------------------------------*/ + +namespace Kokkos { +namespace Impl { + +template<> +struct VerifyExecutionCanAccessMemorySpace + < Kokkos::CudaSpace + , Kokkos::Cuda::scratch_memory_space + > +{ + enum { value = true }; + KOKKOS_INLINE_FUNCTION static void verify( void ) { } + KOKKOS_INLINE_FUNCTION static void verify( const void * ) { } +}; + +template<> +struct VerifyExecutionCanAccessMemorySpace + < Kokkos::HostSpace + , Kokkos::Cuda::scratch_memory_space + > +{ + enum { value = false }; + inline static void verify( void ) { CudaSpace::access_error(); } + inline static void verify( const void * p ) { CudaSpace::access_error(p); } +}; + +} // namespace Impl +} // namespace Kokkos + +/*--------------------------------------------------------------------------*/ +/*--------------------------------------------------------------------------*/ + +#include +#include +#include + +//---------------------------------------------------------------------------- + +#endif /* #if defined( KOKKOS_HAVE_CUDA ) */ +#endif /* #ifndef KOKKOS_CUDA_HPP */ + + + diff --git a/lib/kokkos/core/src/Kokkos_CudaSpace.hpp b/lib/kokkos/core/src/Kokkos_CudaSpace.hpp new file mode 100755 index 0000000000..2c4686126e --- /dev/null +++ b/lib/kokkos/core/src/Kokkos_CudaSpace.hpp @@ -0,0 +1,468 @@ +/* +//@HEADER +// ************************************************************************ +// +// Kokkos: Manycore Performance-Portable Multidimensional Arrays +// Copyright (2012) Sandia Corporation +// +// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +// the U.S. Government retains certain rights in this software. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// 1. Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// 3. Neither the name of the Corporation nor the names of the +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Questions? Contact H. Carter Edwards (hcedwar@sandia.gov) +// +// ************************************************************************ +//@HEADER +*/ + +#ifndef KOKKOS_CUDASPACE_HPP +#define KOKKOS_CUDASPACE_HPP + +#if defined( KOKKOS_HAVE_CUDA ) + +#include +#include +#include + +#include +#include +#include + +/*--------------------------------------------------------------------------*/ + +namespace Kokkos { + +/** \brief Cuda on-device memory management */ + +class CudaSpace { +public: + + //! Tag this class as a kokkos memory space + typedef CudaSpace memory_space ; + typedef Kokkos::Cuda execution_space ; + typedef unsigned int size_type ; + + /** \brief Allocate a contiguous block of memory on the Cuda device. + * + * The input label is associated with the block of memory. + * The block of memory is tracked via reference counting where + * allocation gives it a reference count of one. + * + * Allocation may only occur on the master thread of the process. + */ + static void * allocate( const std::string & label , const size_t size ); + + /** \brief Increment the reference count of the block of memory + * in which the input pointer resides. + * + * Reference counting only occurs on the master thread. + */ + static void increment( const void * ); + + /** \brief Decrement the reference count of the block of memory + * in which the input pointer resides. If the reference + * count falls to zero the memory is deallocated. + * + * Reference counting only occurs on the master thread. + */ + static void decrement( const void * ); + + /** \brief Get the reference count of the block of memory + * in which the input pointer resides. If the reference + * count is zero the memory region is not tracked. + * + * Reference counting only occurs on the master thread. + */ + static int count( const void * ); + + /** \brief Print all tracked memory to the output stream. */ + static void print_memory_view( std::ostream & ); + + /** \brief Retrieve label associated with the input pointer */ + static std::string query_label( const void * ); + + /*--------------------------------*/ + /** \brief Cuda specific function to attached texture object to an allocation. + * Output the texture object, base pointer, and offset from the input pointer. + */ +#if defined( __CUDACC__ ) + static void texture_object_attach( const void * const arg_ptr + , const unsigned arg_type_size + , ::cudaChannelFormatDesc const & arg_desc + , ::cudaTextureObject_t * const arg_tex_obj + , void const ** const arg_alloc_ptr + , int * const arg_offset + ); +#endif + + /*--------------------------------*/ + /** \brief Error reporting for HostSpace attempt to access CudaSpace */ + static void access_error(); + static void access_error( const void * const ); +}; + +} // namespace Kokkos + +/*--------------------------------------------------------------------------*/ +/*--------------------------------------------------------------------------*/ + +namespace Kokkos { + +/** \brief Cuda memory that is accessible to Host execution space + * through Cuda's unified virtual memory (UVM) runtime. + */ +class CudaUVMSpace { +public: + + //! Tag this class as a kokkos memory space + typedef CudaUVMSpace memory_space ; + typedef Cuda execution_space ; + typedef unsigned int size_type ; + + /** \brief If UVM capability is available */ + static bool available(); + + /** \brief Allocate a contiguous block of memory on the Cuda device. + * + * The input label is associated with the block of memory. + * The block of memory is tracked via reference counting where + * allocation gives it a reference count of one. + * + * Allocation may only occur on the master thread of the process. + */ + static void * allocate( const std::string & label , const size_t size ); + + /** \brief Increment the reference count of the block of memory + * in which the input pointer resides. + * + * Reference counting only occurs on the master thread. + */ + static void increment( const void * ); + + /** \brief Decrement the reference count of the block of memory + * in which the input pointer resides. If the reference + * count falls to zero the memory is deallocated. + * + * Reference counting only occurs on the master thread. + */ + static void decrement( const void * ); + + /** \brief Get the reference count of the block of memory + * in which the input pointer resides. If the reference + * count is zero the memory region is not tracked. + * + * Reference counting only occurs on the master thread. + */ + static int count( const void * ); + + /** \brief Print all tracked memory to the output stream. */ + static void print_memory_view( std::ostream & ); + + /** \brief Retrieve label associated with the input pointer */ + static std::string query_label( const void * ); + + /** \brief Cuda specific function to attached texture object to an allocation. + * Output the texture object, base pointer, and offset from the input pointer. + */ +#if defined( __CUDACC__ ) + static void texture_object_attach( const void * const arg_ptr + , const unsigned arg_type_size + , ::cudaChannelFormatDesc const & arg_desc + , ::cudaTextureObject_t * const arg_tex_obj + , void const ** const arg_alloc_ptr + , int * const arg_offset + ); +#endif +}; + +} // namespace Kokkos + +/*--------------------------------------------------------------------------*/ +/*--------------------------------------------------------------------------*/ + +namespace Kokkos { + +/** \brief Host memory that is accessible to Cuda execution space + * through Cuda's host-pinned memory allocation. + */ +class CudaHostPinnedSpace { +public: + + //! Tag this class as a kokkos memory space + typedef CudaHostPinnedSpace memory_space ; + typedef unsigned int size_type ; + + /** \brief Memory is in HostSpace so use the HostSpace::execution_space */ + typedef HostSpace::execution_space execution_space ; + + /** \brief Allocate a contiguous block of memory on the Cuda device. + * + * The input label is associated with the block of memory. + * The block of memory is tracked via reference counting where + * allocation gives it a reference count of one. + * + * Allocation may only occur on the master thread of the process. + */ + static void * allocate( const std::string & label , const size_t size ); + + /** \brief Increment the reference count of the block of memory + * in which the input pointer resides. + * + * Reference counting only occurs on the master thread. + */ + static void increment( const void * ); + + /** \brief Get the reference count of the block of memory + * in which the input pointer resides. If the reference + * count is zero the memory region is not tracked. + * + * Reference counting only occurs on the master thread. + */ + static int count( const void * ); + + /** \brief Decrement the reference count of the block of memory + * in which the input pointer resides. If the reference + * count falls to zero the memory is deallocated. + * + * Reference counting only occurs on the master thread. + */ + static void decrement( const void * ); + + /** \brief Print all tracked memory to the output stream. */ + static void print_memory_view( std::ostream & ); + + /** \brief Retrieve label associated with the input pointer */ + static std::string query_label( const void * ); +}; + +} // namespace Kokkos + +/*--------------------------------------------------------------------------*/ +/*--------------------------------------------------------------------------*/ + +namespace Kokkos { +namespace Impl { + +template<> struct DeepCopy< CudaSpace , CudaSpace > +{ + DeepCopy( void * dst , const void * src , size_t ); + DeepCopy( const Cuda & , void * dst , const void * src , size_t ); +}; + +template<> struct DeepCopy< CudaSpace , HostSpace > +{ + DeepCopy( void * dst , const void * src , size_t ); + DeepCopy( const Cuda & , void * dst , const void * src , size_t ); +}; + +template<> struct DeepCopy< HostSpace , CudaSpace > +{ + DeepCopy( void * dst , const void * src , size_t ); + DeepCopy( const Cuda & , void * dst , const void * src , size_t ); +}; + +template<> struct DeepCopy< CudaSpace , CudaUVMSpace > +{ + inline + DeepCopy( void * dst , const void * src , size_t n ) + { (void) DeepCopy< CudaSpace , CudaSpace >( dst , src , n ); } +}; + +template<> struct DeepCopy< CudaSpace , CudaHostPinnedSpace > +{ + inline + DeepCopy( void * dst , const void * src , size_t n ) + { (void) DeepCopy< CudaSpace , HostSpace >( dst , src , n ); } +}; + + +template<> struct DeepCopy< CudaUVMSpace , CudaSpace > +{ + inline + DeepCopy( void * dst , const void * src , size_t n ) + { (void) DeepCopy< CudaSpace , CudaSpace >( dst , src , n ); } +}; + +template<> struct DeepCopy< CudaUVMSpace , CudaUVMSpace > +{ + inline + DeepCopy( void * dst , const void * src , size_t n ) + { (void) DeepCopy< CudaSpace , CudaSpace >( dst , src , n ); } +}; + +template<> struct DeepCopy< CudaUVMSpace , CudaHostPinnedSpace > +{ + inline + DeepCopy( void * dst , const void * src , size_t n ) + { (void) DeepCopy< CudaSpace , HostSpace >( dst , src , n ); } +}; + +template<> struct DeepCopy< CudaUVMSpace , HostSpace > +{ + inline + DeepCopy( void * dst , const void * src , size_t n ) + { (void) DeepCopy< CudaSpace , HostSpace >( dst , src , n ); } +}; + + +template<> struct DeepCopy< CudaHostPinnedSpace , CudaSpace > +{ + inline + DeepCopy( void * dst , const void * src , size_t n ) + { (void) DeepCopy< HostSpace , CudaSpace >( dst , src , n ); } +}; + +template<> struct DeepCopy< CudaHostPinnedSpace , CudaUVMSpace > +{ + inline + DeepCopy( void * dst , const void * src , size_t n ) + { (void) DeepCopy< HostSpace , CudaSpace >( dst , src , n ); } +}; + +template<> struct DeepCopy< CudaHostPinnedSpace , CudaHostPinnedSpace > +{ + inline + DeepCopy( void * dst , const void * src , size_t n ) + { (void) DeepCopy< HostSpace , HostSpace >( dst , src , n ); } +}; + +template<> struct DeepCopy< CudaHostPinnedSpace , HostSpace > +{ + inline + DeepCopy( void * dst , const void * src , size_t n ) + { (void) DeepCopy< HostSpace , HostSpace >( dst , src , n ); } +}; + + +template<> struct DeepCopy< HostSpace , CudaUVMSpace > +{ + inline + DeepCopy( void * dst , const void * src , size_t n ) + { (void) DeepCopy< HostSpace , CudaSpace >( dst , src , n ); } +}; + +template<> struct DeepCopy< HostSpace , CudaHostPinnedSpace > +{ + inline + DeepCopy( void * dst , const void * src , size_t n ) + { (void) DeepCopy< HostSpace , HostSpace >( dst , src , n ); } +}; + +} // namespace Impl +} // namespace Kokkos + +//---------------------------------------------------------------------------- +//---------------------------------------------------------------------------- + +namespace Kokkos { +namespace Impl { + +/** Running in CudaSpace attempting to access HostSpace: error */ +template<> +struct VerifyExecutionCanAccessMemorySpace< Kokkos::CudaSpace , Kokkos::HostSpace > +{ + enum { value = false }; + KOKKOS_INLINE_FUNCTION static void verify( void ) + { Kokkos::abort("Cuda code attempted to access HostSpace memory"); } + + KOKKOS_INLINE_FUNCTION static void verify( const void * ) + { Kokkos::abort("Cuda code attempted to access HostSpace memory"); } +}; + +/** Running in CudaSpace accessing CudaUVMSpace: ok */ +template<> +struct VerifyExecutionCanAccessMemorySpace< Kokkos::CudaSpace , Kokkos::CudaUVMSpace > +{ + enum { value = true }; + KOKKOS_INLINE_FUNCTION static void verify( void ) { } + KOKKOS_INLINE_FUNCTION static void verify( const void * ) { } +}; + +/** Running in CudaSpace accessing CudaHostPinnedSpace: ok */ +template<> +struct VerifyExecutionCanAccessMemorySpace< Kokkos::CudaSpace , Kokkos::CudaHostPinnedSpace > +{ + enum { value = true }; + KOKKOS_INLINE_FUNCTION static void verify( void ) { } + KOKKOS_INLINE_FUNCTION static void verify( const void * ) { } +}; + +/** Running in CudaSpace attempting to access an unknown space: error */ +template< class OtherSpace > +struct VerifyExecutionCanAccessMemorySpace< + typename enable_if< ! is_same::value , Kokkos::CudaSpace >::type , + OtherSpace > +{ + enum { value = false }; + KOKKOS_INLINE_FUNCTION static void verify( void ) + { Kokkos::abort("Cuda code attempted to access unknown Space memory"); } + + KOKKOS_INLINE_FUNCTION static void verify( const void * ) + { Kokkos::abort("Cuda code attempted to access unknown Space memory"); } +}; + +//---------------------------------------------------------------------------- +/** Running in HostSpace attempting to access CudaSpace */ +template<> +struct VerifyExecutionCanAccessMemorySpace< Kokkos::HostSpace , Kokkos::CudaSpace > +{ + enum { value = false }; + inline static void verify( void ) { CudaSpace::access_error(); } + inline static void verify( const void * p ) { CudaSpace::access_error(p); } +}; + +/** Running in HostSpace accessing CudaUVMSpace is OK */ +template<> +struct VerifyExecutionCanAccessMemorySpace< Kokkos::HostSpace , Kokkos::CudaUVMSpace > +{ + enum { value = true }; + inline static void verify( void ) { } + inline static void verify( const void * ) { } +}; + +/** Running in HostSpace accessing CudaHostPinnedSpace is OK */ +template<> +struct VerifyExecutionCanAccessMemorySpace< Kokkos::HostSpace , Kokkos::CudaHostPinnedSpace > +{ + enum { value = true }; + KOKKOS_INLINE_FUNCTION static void verify( void ) {} + KOKKOS_INLINE_FUNCTION static void verify( const void * ) {} +}; + +//---------------------------------------------------------------------------- + +} // namespace Impl +} // namespace Kokkos + +//---------------------------------------------------------------------------- +//---------------------------------------------------------------------------- + +#endif /* #if defined( KOKKOS_HAVE_CUDA ) */ +#endif /* #define KOKKOS_CUDASPACE_HPP */ + diff --git a/lib/kokkos/core/src/Kokkos_CudaTypes.hpp b/lib/kokkos/core/src/Kokkos_CudaTypes.hpp new file mode 100755 index 0000000000..899e7e1fa5 --- /dev/null +++ b/lib/kokkos/core/src/Kokkos_CudaTypes.hpp @@ -0,0 +1,139 @@ +/* +//@HEADER +// ************************************************************************ +// +// Kokkos: Manycore Performance-Portable Multidimensional Arrays +// Copyright (2012) Sandia Corporation +// +// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +// the U.S. Government retains certain rights in this software. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// 1. Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// 3. Neither the name of the Corporation nor the names of the +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Questions? Contact H. Carter Edwards (hcedwar@sandia.gov) +// +// ************************************************************************ +//@HEADER +*/ + +#ifndef KOKKOS_CUDATYPES_HPP +#define KOKKOS_CUDATYPES_HPP + +#include + +//---------------------------------------------------------------------------- +//---------------------------------------------------------------------------- + +#if defined( __CUDACC__ ) + +namespace Kokkos { + +typedef ::int2 int2 ; +typedef ::int3 int3 ; +typedef ::int4 int4 ; + +typedef ::float2 float2 ; +typedef ::float3 float3 ; +typedef ::float4 float4 ; + +typedef ::double2 double2 ; +typedef ::double3 double3 ; +typedef ::double4 double4 ; + +} // namespace Kokkos + +//---------------------------------------------------------------------------- +//---------------------------------------------------------------------------- + +#else /* NOT #if defined( __CUDACC__ ) */ + +namespace Kokkos { + +struct int2 { + int x; + int y; +}; + +struct int3 { + int x; + int y; + int z; +}; + +struct int4 { + int x; + int y; + int z; + int w; +}; + +struct float2 { + float x; + float y; +}; + +struct float3 { + float x; + float y; + float z; +}; + +struct float4 { + float x; + float y; + float z; + float w; +}; + +struct double2 { + double x; + double y; +}; + +struct double3 { + double x; + double y; + double z; +}; + +struct double4 { + double x; + double y; + double z; + double w; +}; + +} // namespace Kokkos + +#endif + +//---------------------------------------------------------------------------- +//---------------------------------------------------------------------------- + +#endif /* #define KOKKOS_CUDATYPES_HPP */ + diff --git a/lib/kokkos/core/src/Kokkos_ExecPolicy.hpp b/lib/kokkos/core/src/Kokkos_ExecPolicy.hpp new file mode 100755 index 0000000000..209ea4a50c --- /dev/null +++ b/lib/kokkos/core/src/Kokkos_ExecPolicy.hpp @@ -0,0 +1,439 @@ +/* +//@HEADER +// ************************************************************************ +// +// Kokkos: Manycore Performance-Portable Multidimensional Arrays +// Copyright (2012) Sandia Corporation +// +// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +// the U.S. Government retains certain rights in this software. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// 1. Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// 3. Neither the name of the Corporation nor the names of the +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Questions? Contact H. Carter Edwards (hcedwar@sandia.gov) +// +// ************************************************************************ +//@HEADER +*/ + +#ifndef KOKKOS_EXECPOLICY_HPP +#define KOKKOS_EXECPOLICY_HPP + +#include +#include +#include +#include + +//---------------------------------------------------------------------------- + +namespace Kokkos { + +/** \brief Execution policy for work over a range of an integral type. + * + * Valid template argument options: + * + * With a specified execution space: + * < ExecSpace , WorkTag , { IntConst | IntType } > + * < ExecSpace , WorkTag , void > + * < ExecSpace , { IntConst | IntType } , void > + * < ExecSpace , void , void > + * + * With the default execution space: + * < WorkTag , { IntConst | IntType } , void > + * < WorkTag , void , void > + * < { IntConst | IntType } , void , void > + * < void , void , void > + * + * IntType is a fundamental integral type + * IntConst is an Impl::integral_constant< IntType , Blocking > + * + * Blocking is the granularity of partitioning the range among threads. + */ +template< class Arg0 = void , class Arg1 = void , class Arg2 = void + , class ExecSpace = + // The first argument is the execution space, + // otherwise use the default execution space. + typename Impl::if_c< Impl::is_execution_space< Arg0 >::value , Arg0 + , Kokkos::DefaultExecutionSpace >::type + > +class RangePolicy { +private: + + // Default integral type and blocking factor: + typedef int DefaultIntType ; + enum { DefaultIntValue = 8 }; + + enum { Arg0_Void = Impl::is_same< Arg0 , void >::value }; + enum { Arg1_Void = Impl::is_same< Arg1 , void >::value }; + enum { Arg2_Void = Impl::is_same< Arg2 , void >::value }; + + enum { Arg0_ExecSpace = Impl::is_execution_space< Arg0 >::value }; + + enum { Arg0_IntConst = Impl::is_integral_constant< Arg0 >::value }; + enum { Arg1_IntConst = Impl::is_integral_constant< Arg1 >::value }; + enum { Arg2_IntConst = Impl::is_integral_constant< Arg2 >::value }; + + enum { Arg0_IntType = Impl::is_integral< Arg0 >::value }; + enum { Arg1_IntType = Impl::is_integral< Arg1 >::value }; + enum { Arg2_IntType = Impl::is_integral< Arg2 >::value }; + + enum { Arg0_WorkTag = ! Arg0_ExecSpace && ! Arg0_IntConst && ! Arg0_IntType && ! Arg0_Void }; + enum { Arg1_WorkTag = Arg0_ExecSpace && ! Arg1_IntConst && ! Arg1_IntType && ! Arg1_Void }; + + enum { ArgOption_OK = Impl::StaticAssert< ( + ( Arg0_ExecSpace && Arg1_WorkTag && ( Arg2_IntConst || Arg2_IntType ) ) || + ( Arg0_ExecSpace && Arg1_WorkTag && Arg2_Void ) || + ( Arg0_ExecSpace && ( Arg1_IntConst || Arg1_IntType ) && Arg2_Void ) || + ( Arg0_ExecSpace && Arg1_Void && Arg2_Void ) || + ( Arg0_WorkTag && ( Arg1_IntConst || Arg1_IntType ) && Arg2_Void ) || + ( Arg0_WorkTag && Arg1_Void && Arg2_Void ) || + ( ( Arg0_IntConst || Arg0_IntType ) && Arg1_Void && Arg2_Void ) || + ( Arg0_Void && Arg1_Void && Arg2_Void ) + ) >::value }; + + // The work argument tag is the first or second argument + typedef typename Impl::if_c< Arg0_WorkTag , Arg0 , + typename Impl::if_c< Arg1_WorkTag , Arg1 , void + >::type >::type + WorkTag ; + + enum { Granularity = Arg0_IntConst ? unsigned(Impl::is_integral_constant::integral_value) : ( + Arg1_IntConst ? unsigned(Impl::is_integral_constant::integral_value) : ( + Arg2_IntConst ? unsigned(Impl::is_integral_constant::integral_value) : ( + unsigned(DefaultIntValue) ))) }; + + // Only accept the integral type if the blocking is a power of two + typedef typename Impl::enable_if< Impl::is_power_of_two< Granularity >::value , + typename Impl::if_c< Arg0_IntType , Arg0 , + typename Impl::if_c< Arg1_IntType , Arg1 , + typename Impl::if_c< Arg2_IntType , Arg2 , + typename Impl::if_c< Arg0_IntConst , typename Impl::is_integral_constant::integral_type , + typename Impl::if_c< Arg1_IntConst , typename Impl::is_integral_constant::integral_type , + typename Impl::if_c< Arg2_IntConst , typename Impl::is_integral_constant::integral_type , + DefaultIntType + >::type >::type >::type + >::type >::type >::type + >::type + IntType ; + + enum { GranularityMask = IntType(Granularity) - 1 }; + + ExecSpace m_space ; + IntType m_begin ; + IntType m_end ; + +public: + + //! Tag this class as an execution policy + typedef ExecSpace execution_space ; + typedef RangePolicy execution_policy ; + typedef WorkTag work_tag ; + typedef IntType member_type ; + + KOKKOS_INLINE_FUNCTION const execution_space & space() const { return m_space ; } + KOKKOS_INLINE_FUNCTION member_type begin() const { return m_begin ; } + KOKKOS_INLINE_FUNCTION member_type end() const { return m_end ; } + + inline RangePolicy() : m_space(), m_begin(0), m_end(0) {} + + /** \brief Total range */ + inline + RangePolicy( const member_type work_begin + , const member_type work_end + ) + : m_space() + , m_begin( work_begin < work_end ? work_begin : 0 ) + , m_end( work_begin < work_end ? work_end : 0 ) + {} + + /** \brief Total range */ + inline + RangePolicy( const execution_space & work_space + , const member_type work_begin + , const member_type work_end + ) + : m_space( work_space ) + , m_begin( work_begin < work_end ? work_begin : 0 ) + , m_end( work_begin < work_end ? work_end : 0 ) + {} + + /** \brief Subrange for a partition's rank and size. + * + * Typically used to partition a range over a group of threads. + */ + struct WorkRange { + typedef RangePolicy::work_tag work_tag ; + typedef RangePolicy::member_type member_type ; + + KOKKOS_INLINE_FUNCTION member_type begin() const { return m_begin ; } + KOKKOS_INLINE_FUNCTION member_type end() const { return m_end ; } + + /** \brief Subrange for a partition's rank and size. + * + * Typically used to partition a range over a group of threads. + */ + KOKKOS_INLINE_FUNCTION + WorkRange( const RangePolicy & range + , const int part_rank + , const int part_size + ) + : m_begin(0), m_end(0) + { + if ( part_size ) { + + // Split evenly among partitions, then round up to the granularity. + const member_type work_part = + ( ( ( ( range.end() - range.begin() ) + ( part_size - 1 ) ) / part_size ) + + GranularityMask ) & ~member_type(GranularityMask); + + m_begin = range.begin() + work_part * part_rank ; + m_end = m_begin + work_part ; + + if ( range.end() < m_begin ) m_begin = range.end() ; + if ( range.end() < m_end ) m_end = range.end() ; + } + } + private: + member_type m_begin ; + member_type m_end ; + WorkRange(); + WorkRange & operator = ( const WorkRange & ); + }; +}; + +} // namespace Kokkos + +//---------------------------------------------------------------------------- +//---------------------------------------------------------------------------- + +namespace Kokkos { + +/** \brief Execution policy for parallel work over a league of teams of threads. + * + * The work functor is called for each thread of each team such that + * the team's member threads are guaranteed to be concurrent. + * + * The team's threads have access to team shared scratch memory and + * team collective operations. + * + * If the WorkTag is non-void then the first calling argument of the + * work functor's parentheses operator is 'const WorkTag &'. + * This allows a functor to have multiple work member functions. + * + * template argument option with specified execution space: + * < ExecSpace , WorkTag > + * < ExecSpace , void > + * + * template argument option with default execution space: + * < WorkTag , void > + * < void , void > + */ +template< class Arg0 = void + , class Arg1 = void + , class ExecSpace = + // If the first argument is not an execution + // then use the default execution space. + typename Impl::if_c< Impl::is_execution_space< Arg0 >::value , Arg0 + , Kokkos::DefaultExecutionSpace >::type + > +class TeamPolicy { +private: + + enum { Arg0_ExecSpace = Impl::is_execution_space< Arg0 >::value }; + enum { Arg1_Void = Impl::is_same< Arg1 , void >::value }; + enum { ArgOption_OK = Impl::StaticAssert< ( Arg0_ExecSpace || Arg1_Void ) >::value }; + + typedef typename Impl::if_c< Arg0_ExecSpace , Arg1 , Arg0 >::type WorkTag ; + +public: + + //! Tag this class as an execution policy + typedef TeamPolicy execution_policy ; + typedef ExecSpace execution_space ; + typedef WorkTag work_tag ; + + //---------------------------------------- + /** \brief Query maximum team size for a given functor. + * + * This size takes into account execution space concurrency limitations and + * scratch memory space limitations for reductions, team reduce/scan, and + * team shared memory. + */ + template< class FunctorType > + static int team_size_max( const FunctorType & ); + + /** \brief Query recommended team size for a given functor. + * + * This size takes into account execution space concurrency limitations and + * scratch memory space limitations for reductions, team reduce/scan, and + * team shared memory. + */ + template< class FunctorType > + static int team_size_recommended( const FunctorType & ); + + //---------------------------------------- + /** \brief Construct policy with the given instance of the execution space */ + TeamPolicy( const execution_space & , int league_size_request , int team_size_request ); + + /** \brief Construct policy with the default instance of the execution space */ + TeamPolicy( int league_size_request , int team_size_request ); + + /** \brief The actual league size (number of teams) of the policy. + * + * This may be smaller than the requested league size due to limitations + * of the execution space. + */ + KOKKOS_INLINE_FUNCTION int league_size() const ; + + /** \brief The actual team size (number of threads per team) of the policy. + * + * This may be smaller than the requested team size due to limitations + * of the execution space. + */ + KOKKOS_INLINE_FUNCTION int team_size() const ; + + /** \brief Parallel execution of a functor calls the functor once with + * each member of the execution policy. + */ + struct member_type { + + /** \brief Handle to the currently executing team shared scratch memory */ + KOKKOS_INLINE_FUNCTION + typename execution_space::scratch_memory_space team_shmem() const ; + + /** \brief Rank of this team within the league of teams */ + KOKKOS_INLINE_FUNCTION int league_rank() const ; + + /** \brief Number of teams in the league */ + KOKKOS_INLINE_FUNCTION int league_size() const ; + + /** \brief Rank of this thread within this team */ + KOKKOS_INLINE_FUNCTION int team_rank() const ; + + /** \brief Number of threads in this team */ + KOKKOS_INLINE_FUNCTION int team_size() const ; + + /** \brief Barrier among the threads of this team */ + KOKKOS_INLINE_FUNCTION void team_barrier() const ; + + /** \brief Intra-team reduction. Returns join of all values of the team members. */ + template< class JoinOp > + KOKKOS_INLINE_FUNCTION + typename JoinOp::value_type team_reduce( const typename JoinOp::value_type + , const JoinOp & ) const ; + + /** \brief Intra-team exclusive prefix sum with team_rank() ordering. + * + * The highest rank thread can compute the reduction total as + * reduction_total = dev.team_scan( value ) + value ; + */ + template< typename Type > + KOKKOS_INLINE_FUNCTION Type team_scan( const Type & value ) const ; + + /** \brief Intra-team exclusive prefix sum with team_rank() ordering + * with intra-team non-deterministic ordering accumulation. + * + * The global inter-team accumulation value will, at the end of the + * league's parallel execution, be the scan's total. + * Parallel execution ordering of the league's teams is non-deterministic. + * As such the base value for each team's scan operation is similarly + * non-deterministic. + */ + template< typename Type > + KOKKOS_INLINE_FUNCTION Type team_scan( const Type & value , Type * const global_accum ) const ; + }; +}; + +} // namespace Kokkos + +namespace Kokkos { + +namespace Impl { + template + struct TeamThreadLoopBoundariesStruct { + typedef iType index_type; + const iType start; + const iType end; + enum {increment = 1}; + const TeamMemberType& thread; + + KOKKOS_INLINE_FUNCTION + TeamThreadLoopBoundariesStruct (const TeamMemberType& thread_, const iType& count): + start( ( (count + thread_.team_size()-1) / thread_.team_size() ) * thread_.team_rank() ), + end( ( (count + thread_.team_size()-1) / thread_.team_size() ) * ( thread_.team_rank() + 1 ) <= count? + ( (count + thread_.team_size()-1) / thread_.team_size() ) * ( thread_.team_rank() + 1 ):count), + thread(thread_) + {} + }; + + template + struct ThreadVectorLoopBoundariesStruct { + typedef iType index_type; + enum {start = 0}; + const iType end; + enum {increment = 1}; + + KOKKOS_INLINE_FUNCTION + ThreadVectorLoopBoundariesStruct (const TeamMemberType& thread, const iType& count): + end( count ) + {} + }; + + template + struct ThreadSingleStruct { + const TeamMemberType& team_member; + KOKKOS_INLINE_FUNCTION + ThreadSingleStruct(const TeamMemberType& team_member_):team_member(team_member_){} + }; + + template + struct VectorSingleStruct { + const TeamMemberType& team_member; + KOKKOS_INLINE_FUNCTION + VectorSingleStruct(const TeamMemberType& team_member_):team_member(team_member_){} + }; +} // namespace Impl + +/*template +KOKKOS_INLINE_FUNCTION +Impl::TeamThreadLoopBoundariesStruct + TeamThreadLoop(TeamMemberType thread, const iType count); + +template +KOKKOS_INLINE_FUNCTION +Impl::ThreadVectorLoopBoundariesStruct + ThreadVectorLoop(TeamMemberType thread, const iType count);*/ + + +} // namespace Kokkos + +#endif /* #define KOKKOS_EXECPOLICY_HPP */ + +//---------------------------------------------------------------------------- +//---------------------------------------------------------------------------- + diff --git a/lib/kokkos/core/src/Kokkos_HostSpace.hpp b/lib/kokkos/core/src/Kokkos_HostSpace.hpp new file mode 100755 index 0000000000..80abf0b50d --- /dev/null +++ b/lib/kokkos/core/src/Kokkos_HostSpace.hpp @@ -0,0 +1,161 @@ +/* +//@HEADER +// ************************************************************************ +// +// Kokkos: Manycore Performance-Portable Multidimensional Arrays +// Copyright (2012) Sandia Corporation +// +// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +// the U.S. Government retains certain rights in this software. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// 1. Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// 3. Neither the name of the Corporation nor the names of the +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Questions? Contact H. Carter Edwards (hcedwar@sandia.gov) +// +// ************************************************************************ +//@HEADER +*/ + +#ifndef KOKKOS_HOSTSPACE_HPP +#define KOKKOS_HOSTSPACE_HPP + +#include +#include +#include + +#include +#include +#include +#include + +/*--------------------------------------------------------------------------*/ + +namespace Kokkos { + +/// \class HostSpace +/// \brief Memory management for host memory. +/// +/// HostSpace is a memory space that governs host memory. "Host" +/// memory means the usual CPU-accessible memory. +class HostSpace { +public: + + //! Tag this class as a kokkos memory space + typedef HostSpace memory_space ; + typedef size_t size_type ; + + /// \typedef execution_space + /// \brief Default execution space for this memory space. + /// + /// Every memory space has a default execution space. This is + /// useful for things like initializing a View (which happens in + /// parallel using the View's default execution space). +#if defined( KOKKOS_HAVE_DEFAULT_DEVICE_TYPE_OPENMP ) + typedef Kokkos::OpenMP execution_space ; +#elif defined( KOKKOS_HAVE_DEFAULT_DEVICE_TYPE_THREADS ) + typedef Kokkos::Threads execution_space ; +#elif defined( KOKKOS_HAVE_OPENMP ) + typedef Kokkos::OpenMP execution_space ; +#elif defined( KOKKOS_HAVE_PTHREAD ) + typedef Kokkos::Threads execution_space ; +#elif defined( KOKKOS_HAVE_SERIAL ) + typedef Kokkos::Serial execution_space ; +#else +# error "At least one of the following host execution spaces must be defined: Kokkos::OpenMP, Kokkos::Serial, or Kokkos::Threads. You might be seeing this message if you disabled the Kokkos::Serial device explicitly using the Kokkos_ENABLE_Serial:BOOL=OFF CMake option, but did not enable any of the other host execution space devices." +#endif + + /** \brief Allocate a contiguous block of memory. + * + * The input label is associated with the block of memory. + * The block of memory is tracked via reference counting where + * allocation gives it a reference count of one. + * + * Allocation may only occur on the master thread of the process. + */ + static void * allocate( const std::string & label , const size_t size ); + + /** \brief Increment the reference count of the block of memory + * in which the input pointer resides. + * + * Reference counting only occurs on the master thread. + */ + static void increment( const void * ); + + /** \brief Decrement the reference count of the block of memory + * in which the input pointer resides. If the reference + * count falls to zero the memory is deallocated. + * + * Reference counting only occurs on the master thread. + */ + static void decrement( const void * ); + + /** \brief Get the reference count of the block of memory + * in which the input pointer resides. If the reference + * count is zero the memory region is not tracked. + * + * Reference counting only occurs on the master thread. + */ + static int count( const void * ); + + /*--------------------------------*/ + + /** \brief Print all tracked memory to the output stream. */ + static void print_memory_view( std::ostream & ); + + /** \brief Retrieve label associated with the input pointer */ + static std::string query_label( const void * ); + + /*--------------------------------*/ + /* Functions unique to the HostSpace */ + + static int in_parallel(); + + static void register_in_parallel( int (*)() ); +}; + + +} // namespace Kokkos + +//---------------------------------------------------------------------------- +//---------------------------------------------------------------------------- + +namespace Kokkos { +namespace Impl { + +template< class , class > struct DeepCopy ; + +template<> +struct DeepCopy { + DeepCopy( void * dst , const void * src , size_t n ); +}; + +} // namespace Impl +} // namespace Kokkos + +#endif /* #define KOKKOS_HOSTSPACE_HPP */ + diff --git a/lib/kokkos/core/src/Kokkos_Layout.hpp b/lib/kokkos/core/src/Kokkos_Layout.hpp new file mode 100755 index 0000000000..1440ad84a8 --- /dev/null +++ b/lib/kokkos/core/src/Kokkos_Layout.hpp @@ -0,0 +1,176 @@ +/* +//@HEADER +// ************************************************************************ +// +// Kokkos +// Manycore Performance-Portable Multidimensional Arrays +// +// Copyright (2012) Sandia Corporation +// +// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +// the U.S. Government retains certain rights in this software. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// 1. Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// 3. Neither the name of the Corporation nor the names of the +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Questions? Contact H. Carter Edwards (hcedwar@sandia.gov) +// +// ************************************************************************ +//@HEADER +*/ + +/// \file Kokkos_Layout.hpp +/// \brief Declaration of various \c MemoryLayout options. + +#ifndef KOKKOS_LAYOUT_HPP +#define KOKKOS_LAYOUT_HPP + +#include +#include +#include + +namespace Kokkos { + +//---------------------------------------------------------------------------- +/// \struct LayoutLeft +/// \brief Memory layout tag indicating left-to-right (Fortran scheme) +/// striding of multi-indices. +/// +/// This is an example of a \c MemoryLayout template parameter of +/// View. The memory layout describes how View maps from a +/// multi-index (i0, i1, ..., ik) to a memory location. +/// +/// "Layout left" indicates a mapping where the leftmost index i0 +/// refers to contiguous access, and strides increase for dimensions +/// going right from there (i1, i2, ...). This layout imitates how +/// Fortran stores multi-dimensional arrays. For the special case of +/// a two-dimensional array, "layout left" is also called "column +/// major." +struct LayoutLeft { + //! Tag this class as a kokkos array layout + typedef LayoutLeft array_layout ; +}; + +//---------------------------------------------------------------------------- +/// \struct LayoutRight +/// \brief Memory layout tag indicating right-to-left (C or +/// lexigraphical scheme) striding of multi-indices. +/// +/// This is an example of a \c MemoryLayout template parameter of +/// View. The memory layout describes how View maps from a +/// multi-index (i0, i1, ..., ik) to a memory location. +/// +/// "Right layout" indicates a mapping where the rightmost index ik +/// refers to contiguous access, and strides increase for dimensions +/// going left from there. This layout imitates how C stores +/// multi-dimensional arrays. For the special case of a +/// two-dimensional array, "layout right" is also called "row major." +struct LayoutRight { + //! Tag this class as a kokkos array layout + typedef LayoutRight array_layout ; +}; + +//---------------------------------------------------------------------------- +/// \struct LayoutStride +/// \brief Memory layout tag indicated arbitrarily strided +/// multi-index mapping into contiguous memory. +struct LayoutStride { + + //! Tag this class as a kokkos array layout + typedef LayoutStride array_layout ; + + enum { MAX_RANK = 8 }; + + size_t dimension[ MAX_RANK ] ; + size_t stride[ MAX_RANK ] ; + + /** \brief Compute strides from ordered dimensions. + * + * Values of order uniquely form the set [0..rank) + * and specify ordering of the dimensions. + * Order = {0,1,2,...} is LayoutLeft + * Order = {...,2,1,0} is LayoutRight + */ + template< typename iTypeOrder , typename iTypeDimen > + KOKKOS_INLINE_FUNCTION static + LayoutStride order_dimensions( int const rank + , iTypeOrder const * const order + , iTypeDimen const * const dimen ) + { + LayoutStride tmp ; + // Verify valid rank order: + int check_input = MAX_RANK < rank ? 0 : int( 1 << rank ) - 1 ; + for ( int r = 0 ; r < MAX_RANK ; ++r ) { + tmp.dimension[r] = 0 ; + tmp.stride[r] = 0 ; + check_input &= ~int( 1 << order[r] ); + } + if ( 0 == check_input ) { + size_t n = 1 ; + for ( int r = 0 ; r < rank ; ++r ) { + tmp.stride[ order[r] ] = n ; + n *= ( dimen[order[r]] ); + tmp.dimension[r] = dimen[r]; + } + } + return tmp ; + } +}; + +//---------------------------------------------------------------------------- +/// \struct LayoutTileLeft +/// \brief Memory layout tag indicating left-to-right (Fortran scheme) +/// striding of multi-indices by tiles. +/// +/// This is an example of a \c MemoryLayout template parameter of +/// View. The memory layout describes how View maps from a +/// multi-index (i0, i1, ..., ik) to a memory location. +/// +/// "Tiled layout" indicates a mapping to contiguously stored +/// ArgN0 by ArgN1 tiles for the rightmost two +/// dimensions. Indices are LayoutLeft within each tile, and the +/// tiles themselves are arranged using LayoutLeft. Note that the +/// dimensions ArgN0 and ArgN1 of the tiles must be +/// compile-time constants. This speeds up index calculations. If +/// both tile dimensions are powers of two, Kokkos can optimize +/// further. +template < unsigned ArgN0 , unsigned ArgN1 , + bool IsPowerOfTwo = ( Impl::is_power_of_two::value && + Impl::is_power_of_two::value ) + > +struct LayoutTileLeft { + //! Tag this class as a kokkos array layout + typedef LayoutTileLeft array_layout ; + + enum { N0 = ArgN0 }; + enum { N1 = ArgN1 }; +}; + +} // namespace Kokkos + +#endif // #ifndef KOKKOS_LAYOUT_HPP + diff --git a/lib/kokkos/core/src/Kokkos_Macros.hpp b/lib/kokkos/core/src/Kokkos_Macros.hpp new file mode 100755 index 0000000000..a67aa1adcb --- /dev/null +++ b/lib/kokkos/core/src/Kokkos_Macros.hpp @@ -0,0 +1,433 @@ +/* +//@HEADER +// ************************************************************************ +// +// Kokkos +// Manycore Performance-Portable Multidimensional Arrays +// +// Copyright (2012) Sandia Corporation +// +// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +// the U.S. Government retains certain rights in this software. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// 1. Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// 3. Neither the name of the Corporation nor the names of the +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Questions? Contact H. Carter Edwards (hcedwar@sandia.gov) +// +// ************************************************************************ +//@HEADER +*/ + +#ifndef KOKKOS_MACROS_HPP +#define KOKKOS_MACROS_HPP + +//---------------------------------------------------------------------------- +/** Pick up configure/build options via #define macros: + * + * KOKKOS_HAVE_CUDA Kokkos::Cuda execution and memory spaces + * KOKKOS_HAVE_PTHREAD Kokkos::Threads execution space + * KOKKOS_HAVE_QTHREAD Kokkos::Qthread execution space + * KOKKOS_HAVE_OPENMP Kokkos::OpenMP execution space + * KOKKOS_HAVE_HWLOC HWLOC library is available + * KOKKOS_HAVE_EXPRESSION_CHECK insert array bounds checks, is expensive! + * KOKKOS_HAVE_CXX11 enable C++11 features + * + * KOKKOS_HAVE_MPI negotiate MPI/execution space interactions + * + * KOKKOS_USE_CUDA_UVM Use CUDA UVM for Cuda memory space + */ + +#ifndef KOKKOS_DONT_INCLUDE_CORE_CONFIG_H +#include +#endif + +//---------------------------------------------------------------------------- +/** Pick up compiler specific #define macros: + * + * Macros for known compilers evaluate to an integral version value + * + * KOKKOS_COMPILER_NVCC + * KOKKOS_COMPILER_GNU + * KOKKOS_COMPILER_INTEL + * KOKKOS_COMPILER_IBM + * KOKKOS_COMPILER_CRAYC + * KOKKOS_COMPILER_APPLECC + * KOKKOS_COMPILER_CLANG + * KOKKOS_COMPILER_PGI + * + * Macros for which compiler extension to use for atomics on intrinsice types + * + * KOKKOS_ATOMICS_USE_CUDA + * KOKKOS_ATOMICS_USE_GNU + * KOKKOS_ATOMICS_USE_INTEL + * KOKKOS_ATOMICS_USE_OPENMP31 + * + * A suite of 'KOKKOS_HAVE_PRAGMA_...' are defined for internal use. + * + * Macros for marking functions to run in an execution space: + * + * KOKKOS_FUNCTION + * KOKKOS_INLINE_FUNCTION request compiler to inline + * KOKKOS_FORCEINLINE_FUNCTION force compiler to inline, use with care! + */ + +//---------------------------------------------------------------------------- + +#if defined( KOKKOS_HAVE_CUDA ) && defined( __CUDACC__ ) + +/* Compiling with a CUDA compiler. + * + * Include to pick up the CUDA_VERSION macro defined as: + * CUDA_VERSION = ( MAJOR_VERSION * 1000 ) + ( MINOR_VERSION * 10 ) + * + * When generating device code the __CUDA_ARCH__ macro is defined as: + * __CUDA_ARCH__ = ( MAJOR_CAPABILITY * 100 ) + ( MINOR_CAPABILITY * 10 ) + */ + +#include +#include + +#if ! defined( CUDA_VERSION ) +#error "#include did not define CUDA_VERSION" +#endif + +#if ( CUDA_VERSION < 4010 ) +#error "Cuda version 4.1 or greater required" +#endif + +#if defined( __CUDA_ARCH__ ) && ( __CUDA_ARCH__ < 200 ) +/* Compiling with CUDA compiler for device code. */ +#error "Cuda device capability >= 2.0 is required" +#endif + +#endif /* #if defined( KOKKOS_HAVE_CUDA ) && defined( __CUDACC__ ) */ + +/*--------------------------------------------------------------------------*/ +/* Language info: C++, CUDA, OPENMP */ + +#if defined( __CUDA_ARCH__ ) && defined( KOKKOS_HAVE_CUDA ) + // Compiling Cuda code to 'ptx' + + #define KOKKOS_FORCEINLINE_FUNCTION __device__ __host__ __forceinline__ + #define KOKKOS_INLINE_FUNCTION __device__ __host__ inline + #define KOKKOS_FUNCTION __device__ __host__ + +#endif /* #if defined( __CUDA_ARCH__ ) */ + +#if defined( _OPENMP ) + + /* Compiling with OpenMP. + * The value of _OPENMP is an integer value YYYYMM + * where YYYY and MM are the year and month designation + * of the supported OpenMP API version. + */ + +#endif /* #if defined( _OPENMP ) */ + +/*--------------------------------------------------------------------------*/ +/* Mapping compiler built-ins to KOKKOS_COMPILER_*** macros */ + +#if defined( __NVCC__ ) + // NVIDIA compiler is being used. + // Code is parsed and separated into host and device code. + // Host code is compiled again with another compiler. + // Device code is compile to 'ptx'. + #define KOKKOS_COMPILER_NVCC __NVCC__ + + #if defined( KOKKOS_HAVE_CXX11 ) && defined (KOKKOS_HAVE_CUDA) + // CUDA supports (inofficially) C++11 in device code starting with + // version 6.5. This includes auto type and device code internal + // lambdas. + #if ( CUDA_VERSION < 6050 ) + #error "NVCC does not support C++11" + #endif + #endif +#else + #if defined( KOKKOS_HAVE_CXX11 ) + // CUDA (including version 6.5) does not support giving lambdas as + // arguments to global functions. Thus its not currently possible + // to dispatch lambdas from the host. + #define KOKKOS_HAVE_CXX11_DISPATCH_LAMBDA 1 + #endif +#endif /* #if defined( __NVCC__ ) */ + +#if defined( KOKKOS_HAVE_CXX11 ) && !defined (KOKKOS_LAMBDA) + #define KOKKOS_LAMBDA [=] +#endif + +#if ! defined( __CUDA_ARCH__ ) /* Not compiling Cuda code to 'ptx'. */ + +/* Intel compiler for host code */ + +#if defined( __INTEL_COMPILER ) + #define KOKKOS_COMPILER_INTEL __INTEL_COMPILER +#elif defined( __ICC ) + // Old define + #define KOKKOS_COMPILER_INTEL __ICC +#elif defined( __ECC ) + // Very old define + #define KOKKOS_COMPILER_INTEL __ECC +#endif + +/* CRAY compiler for host code */ +#if defined( _CRAYC ) + #define KOKKOS_COMPILER_CRAYC _CRAYC +#endif + +#if defined( __IBMCPP__ ) + // IBM C++ + #define KOKKOS_COMPILER_IBM __IBMCPP__ +#elif defined( __IBMC__ ) + #define KOKKOS_COMPILER_IBM __IBMC__ +#endif + +#if defined( __APPLE_CC__ ) + #define KOKKOS_COMPILER_APPLECC __APPLE_CC__ +#endif + +#if defined (__clang__) && !defined (KOKKOS_COMPILER_INTEL) + #define KOKKOS_COMPILER_CLANG __clang_major__*100+__clang_minor__*10+__clang_patchlevel__ +#endif + +#if ! defined( __clang__ ) && ! defined( KOKKOS_COMPILER_INTEL ) &&defined( __GNUC__ ) + #define KOKKOS_COMPILER_GNU __GNUC__*100+__GNUC_MINOR__*10+__GNUC_PATCHLEVEL__ +#endif + +#if defined( __PGIC__ ) && ! defined( __GNUC__ ) + #define KOKKOS_COMPILER_PGI __PGIC__*100+__PGIC_MINOR__*10+__PGIC_PATCHLEVEL__ +#endif + +#endif /* #if ! defined( __CUDA_ARCH__ ) */ + +/*--------------------------------------------------------------------------*/ +/*--------------------------------------------------------------------------*/ +/* Intel compiler macros */ + +#if defined( KOKKOS_COMPILER_INTEL ) + + #define KOKKOS_HAVE_PRAGMA_UNROLL 1 + #define KOKKOS_HAVE_PRAGMA_IVDEP 1 + #define KOKKOS_HAVE_PRAGMA_LOOPCOUNT 1 + #define KOKKOS_HAVE_PRAGMA_VECTOR 1 + #define KOKKOS_HAVE_PRAGMA_SIMD 1 + + #if ( 1200 <= KOKKOS_COMPILER_INTEL ) && ! defined( KOKKOS_ENABLE_ASM ) + #define KOKKOS_ENABLE_ASM 1 + #endif + + #if ( 1200 <= KOKKOS_COMPILER_INTEL ) && ! defined( KOKKOS_FORCEINLINE_FUNCTION ) + #define KOKKOS_FORCEINLINE_FUNCTION inline __attribute__((always_inline)) + #endif + + #if defined( __MIC__ ) + // Compiling for Xeon Phi + #endif + +#endif + +/*--------------------------------------------------------------------------*/ +/* Cray compiler macros */ + +#if defined( KOKKOS_COMPILER_CRAYC ) + + +#endif + +/*--------------------------------------------------------------------------*/ +/* IBM Compiler macros */ + +#if defined( KOKKOS_COMPILER_IBM ) + + #define KOKKOS_HAVE_PRAGMA_UNROLL 1 + //#define KOKKOS_HAVE_PRAGMA_IVDEP 1 + //#define KOKKOS_HAVE_PRAGMA_LOOPCOUNT 1 + //#define KOKKOS_HAVE_PRAGMA_VECTOR 1 + //#define KOKKOS_HAVE_PRAGMA_SIMD 1 + +#endif + +/*--------------------------------------------------------------------------*/ +/* CLANG compiler macros */ + +#if defined( KOKKOS_COMPILER_CLANG ) + + //#define KOKKOS_HAVE_PRAGMA_UNROLL 1 + //#define KOKKOS_HAVE_PRAGMA_IVDEP 1 + //#define KOKKOS_HAVE_PRAGMA_LOOPCOUNT 1 + //#define KOKKOS_HAVE_PRAGMA_VECTOR 1 + //#define KOKKOS_HAVE_PRAGMA_SIMD 1 + + #if ! defined( KOKKOS_FORCEINLINE_FUNCTION ) + #define KOKKOS_FORCEINLINE_FUNCTION inline __attribute__((always_inline)) + #endif + +#endif + +/*--------------------------------------------------------------------------*/ +/* GNU Compiler macros */ + +#if defined( KOKKOS_COMPILER_GNU ) + + //#define KOKKOS_HAVE_PRAGMA_UNROLL 1 + //#define KOKKOS_HAVE_PRAGMA_IVDEP 1 + //#define KOKKOS_HAVE_PRAGMA_LOOPCOUNT 1 + //#define KOKKOS_HAVE_PRAGMA_VECTOR 1 + //#define KOKKOS_HAVE_PRAGMA_SIMD 1 + + #if ! defined( KOKKOS_FORCEINLINE_FUNCTION ) + #define KOKKOS_FORCEINLINE_FUNCTION inline __attribute__((always_inline)) + #endif + + #if ! defined( KOKKOS_ENABLE_ASM ) && \ + ! ( defined( __powerpc) || \ + defined(__powerpc__) || \ + defined(__powerpc64__) || \ + defined(__POWERPC__) || \ + defined(__ppc__) || \ + defined(__ppc64__) ) + #define KOKKOS_ENABLE_ASM 1 + #endif + +#endif + +/*--------------------------------------------------------------------------*/ + +#if defined( KOKKOS_COMPILER_PGI ) + + #define KOKKOS_HAVE_PRAGMA_UNROLL 1 + #define KOKKOS_HAVE_PRAGMA_IVDEP 1 + //#define KOKKOS_HAVE_PRAGMA_LOOPCOUNT 1 + #define KOKKOS_HAVE_PRAGMA_VECTOR 1 + //#define KOKKOS_HAVE_PRAGMA_SIMD 1 + +#endif + +/*--------------------------------------------------------------------------*/ + +#if defined( KOKKOS_COMPILER_NVCC ) + + #if defined(__CUDA_ARCH__ ) + #define KOKKOS_HAVE_PRAGMA_UNROLL 1 + #endif + +#endif + +/*--------------------------------------------------------------------------*/ +/* Select compiler dependent interface for atomics */ + +#if ! defined( KOKKOS_ATOMICS_USE_CUDA ) || \ + ! defined( KOKKOS_ATOMICS_USE_GNU ) || \ + ! defined( KOKKOS_ATOMICS_USE_INTEL ) || \ + ! defined( KOKKOS_ATOMICS_USE_OPENMP31 ) + +/* Atomic selection is not pre-defined, choose from language and compiler. */ + +#if defined( __CUDA_ARCH__ ) && defined (KOKKOS_HAVE_CUDA) + + #define KOKKOS_ATOMICS_USE_CUDA + +#elif defined( KOKKOS_COMPILER_GNU ) || defined( KOKKOS_COMPILER_CLANG ) + + #define KOKKOS_ATOMICS_USE_GNU + +#elif defined( KOKKOS_COMPILER_INTEL ) || defined( KOKKOS_COMPILER_CRAYC ) + + #define KOKKOS_ATOMICS_USE_INTEL + +#elif defined( _OPENMP ) && ( 201107 <= _OPENMP ) + + #define KOKKOS_ATOMICS_USE_OMP31 + +#else + + #error "Compiler does not support atomic operations" + +#endif + +#endif + +//---------------------------------------------------------------------------- +/** Define function marking macros if compiler specific macros are undefined: */ + +#if ! defined( KOKKOS_FORCEINLINE_FUNCTION ) +#define KOKKOS_FORCEINLINE_FUNCTION inline +#endif + +#if ! defined( KOKKOS_INLINE_FUNCTION ) +#define KOKKOS_INLINE_FUNCTION inline +#endif + +#if ! defined( KOKKOS_FUNCTION ) +#define KOKKOS_FUNCTION /**/ +#endif + +//---------------------------------------------------------------------------- +/** Determine the default execution space for parallel dispatch. + * There is zero or one default execution space specified. + */ + +#if 1 < ( ( defined ( KOKKOS_HAVE_DEFAULT_DEVICE_TYPE_CUDA ) ? 1 : 0 ) + \ + ( defined ( KOKKOS_HAVE_DEFAULT_DEVICE_TYPE_OPENMP ) ? 1 : 0 ) + \ + ( defined ( KOKKOS_HAVE_DEFAULT_DEVICE_TYPE_THREADS ) ? 1 : 0 ) + \ + ( defined ( KOKKOS_HAVE_DEFAULT_DEVICE_TYPE_SERIAL ) ? 1 : 0 ) ) + +#error "More than one KOKKOS_HAVE_DEFAULT_DEVICE_TYPE_* specified" ; + +#endif + +/** If default is not specified then chose from enabled execution spaces. + * Priority: CUDA, OPENMP, THREADS, SERIAL + */ +#if defined ( KOKKOS_HAVE_DEFAULT_DEVICE_TYPE_CUDA ) +#elif defined ( KOKKOS_HAVE_DEFAULT_DEVICE_TYPE_OPENMP ) +#elif defined ( KOKKOS_HAVE_DEFAULT_DEVICE_TYPE_THREADS ) +#elif defined ( KOKKOS_HAVE_DEFAULT_DEVICE_TYPE_SERIAL ) +#elif defined ( KOKKOS_HAVE_CUDA ) +#define KOKKOS_HAVE_DEFAULT_DEVICE_TYPE_CUDA +#elif defined ( KOKKOS_HAVE_OPENMP ) +#define KOKKOS_HAVE_DEFAULT_DEVICE_TYPE_OPENMP +#elif defined ( KOKKOS_HAVE_PTHREAD ) +#define KOKKOS_HAVE_DEFAULT_DEVICE_TYPE_THREADS +#else +#define KOKKOS_HAVE_DEFAULT_DEVICE_TYPE_SERIAL +#endif + +//---------------------------------------------------------------------------- +/** Determine for what space the code is being compiled: */ + +#if defined( __CUDACC__ ) && defined( __CUDA_ARCH__ ) && defined (KOKKOS_HAVE_CUDA) +#define KOKKOS_ACTIVE_EXECUTION_MEMORY_SPACE_CUDA +#else +#define KOKKOS_ACTIVE_EXECUTION_MEMORY_SPACE_HOST +#endif + +//---------------------------------------------------------------------------- +//---------------------------------------------------------------------------- + +#endif /* #ifndef KOKKOS_MACROS_HPP */ + diff --git a/lib/kokkos/core/src/Kokkos_MemoryTraits.hpp b/lib/kokkos/core/src/Kokkos_MemoryTraits.hpp new file mode 100755 index 0000000000..0817e20a89 --- /dev/null +++ b/lib/kokkos/core/src/Kokkos_MemoryTraits.hpp @@ -0,0 +1,118 @@ +/* +//@HEADER +// ************************************************************************ +// +// Kokkos +// Manycore Performance-Portable Multidimensional Arrays +// +// Copyright (2012) Sandia Corporation +// +// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +// the U.S. Government retains certain rights in this software. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// 1. Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// 3. Neither the name of the Corporation nor the names of the +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Questions? Contact H. Carter Edwards (hcedwar@sandia.gov) +// +// ************************************************************************ +//@HEADER +*/ + +#ifndef KOKKOS_MEMORYTRAITS_HPP +#define KOKKOS_MEMORYTRAITS_HPP + +#include +#include + +//---------------------------------------------------------------------------- + +namespace Kokkos { + +/** \brief Memory access traits for views, an extension point. + * + * These traits should be orthogonal. If there are dependencies then + * the MemoryTraits template must detect and enforce dependencies. + * + * A zero value is the default for a View, indicating that none of + * these traits are present. + */ +enum MemoryTraitsFlags + { Unmanaged = 0x01 + , RandomAccess = 0x02 + , Atomic = 0x04 + }; + +template < unsigned T > +struct MemoryTraits { + //! Tag this class as a kokkos memory traits: + typedef MemoryTraits memory_traits ; + + enum { Unmanaged = T & unsigned(Kokkos::Unmanaged) }; + enum { RandomAccess = T & unsigned(Kokkos::RandomAccess) }; + enum { Atomic = T & unsigned(Kokkos::Atomic) }; + +}; + +} // namespace Kokkos + +//---------------------------------------------------------------------------- + +namespace Kokkos { + +typedef Kokkos::MemoryTraits<0> MemoryManaged ; +typedef Kokkos::MemoryTraits< Kokkos::Unmanaged > MemoryUnmanaged ; +typedef Kokkos::MemoryTraits< Kokkos::Unmanaged | Kokkos::RandomAccess > MemoryRandomAccess ; + +} // namespace Kokkos + +//---------------------------------------------------------------------------- + +namespace Kokkos { +namespace Impl { + +/** \brief Memory alignment settings + * + * Sets global value for memory alignment. Must be a power of two! + * Enable compatibility of views from different devices with static stride. + * Use compiler flag to enable overwrites. + */ +enum { MEMORY_ALIGNMENT = +#if defined( KOKKOS_MEMORY_ALIGNMENT ) + ( 1 << Kokkos::Impl::power_of_two< KOKKOS_MEMORY_ALIGNMENT >::value ) +#else + ( 1 << Kokkos::Impl::power_of_two< 128 >::value ) +#endif + , MEMORY_ALIGNMENT_THRESHOLD = 4 + }; + + +} //namespace Impl +} // namespace Kokkos + +#endif /* #ifndef KOKKOS_MEMORYTRAITS_HPP */ + diff --git a/lib/kokkos/core/src/Kokkos_OpenMP.hpp b/lib/kokkos/core/src/Kokkos_OpenMP.hpp new file mode 100755 index 0000000000..7c75357e5f --- /dev/null +++ b/lib/kokkos/core/src/Kokkos_OpenMP.hpp @@ -0,0 +1,176 @@ +/* +//@HEADER +// ************************************************************************ +// +// Kokkos +// Manycore Performance-Portable Multidimensional Arrays +// +// Copyright (2012) Sandia Corporation +// +// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +// the U.S. Government retains certain rights in this software. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// 1. Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// 3. Neither the name of the Corporation nor the names of the +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Questions? Contact H. Carter Edwards (hcedwar@sandia.gov) +// +// ************************************************************************ +//@HEADER +*/ + +#ifndef KOKKOS_OPENMP_HPP +#define KOKKOS_OPENMP_HPP + +#include + +#if defined( KOKKOS_HAVE_OPENMP ) && defined( _OPENMP ) + +#include + +#include +#include +#include +#include +#include +#include +#include + +/*--------------------------------------------------------------------------*/ + +namespace Kokkos { + +/// \class OpenMP +/// \brief Kokkos device for multicore processors in the host memory space. +class OpenMP { +public: + //------------------------------------ + //! \name Type declarations that all Kokkos devices must provide. + //@{ + + //! Tag this class as a kokkos execution space + typedef OpenMP execution_space ; + typedef HostSpace memory_space ; + typedef LayoutRight array_layout ; + typedef HostSpace::size_type size_type ; + + typedef ScratchMemorySpace< OpenMP > scratch_memory_space ; + + //! For backward compatibility + typedef OpenMP device_type ; + //@} + //------------------------------------ + //! \name Functions that all Kokkos devices must implement. + //@{ + + inline static bool in_parallel() { return omp_in_parallel(); } + + /** \brief Set the device in a "sleep" state. A noop for OpenMP. */ + static bool sleep(); + + /** \brief Wake the device from the 'sleep' state. A noop for OpenMP. */ + static bool wake(); + + /** \brief Wait until all dispatched functors complete. A noop for OpenMP. */ + static void fence() {} + + /// \brief Print configuration information to the given output stream. + static void print_configuration( std::ostream & , const bool detail = false ); + + /// \brief Free any resources being consumed by the device. + static void finalize(); + + /** \brief Initialize the device. + * + * 1) If the hardware locality library is enabled and OpenMP has not + * already bound threads then bind OpenMP threads to maximize + * core utilization and group for memory hierarchy locality. + * + * 2) Allocate a HostThread for each OpenMP thread to hold its + * topology and fan in/out data. + */ + static void initialize( unsigned thread_count = 0 , + unsigned use_numa_count = 0 , + unsigned use_cores_per_numa = 0 ); + + static int is_initialized(); + //@} + //------------------------------------ + /** \brief This execution space has a topological thread pool which can be queried. + * + * All threads within a pool have a common memory space for which they are cache coherent. + * depth = 0 gives the number of threads in the whole pool. + * depth = 1 gives the number of threads in a NUMA region, typically sharing L3 cache. + * depth = 2 gives the number of threads at the finest granularity, typically sharing L1 cache. + */ + inline static int thread_pool_size( int depth = 0 ); + + /** \brief The rank of the executing thread in this thread pool */ + KOKKOS_INLINE_FUNCTION static int thread_pool_rank(); + + //------------------------------------ + + inline static unsigned max_hardware_threads() { return thread_pool_size(0); } + + KOKKOS_INLINE_FUNCTION static + unsigned hardware_thread_id() { return thread_pool_rank(); } +}; + +} // namespace Kokkos + +/*--------------------------------------------------------------------------*/ +/*--------------------------------------------------------------------------*/ + +namespace Kokkos { +namespace Impl { + +template<> +struct VerifyExecutionCanAccessMemorySpace + < Kokkos::OpenMP::memory_space + , Kokkos::OpenMP::scratch_memory_space + > +{ + enum { value = true }; + inline static void verify( void ) { } + inline static void verify( const void * ) { } +}; + +} // namespace Impl +} // namespace Kokkos + +/*--------------------------------------------------------------------------*/ +/*--------------------------------------------------------------------------*/ + +#include +#include + +/*--------------------------------------------------------------------------*/ + +#endif /* #if defined( KOKKOS_HAVE_OPENMP ) && defined( _OPENMP ) */ +#endif /* #ifndef KOKKOS_OPENMP_HPP */ + + diff --git a/lib/kokkos/core/src/Kokkos_Pair.hpp b/lib/kokkos/core/src/Kokkos_Pair.hpp new file mode 100755 index 0000000000..c69273cb86 --- /dev/null +++ b/lib/kokkos/core/src/Kokkos_Pair.hpp @@ -0,0 +1,457 @@ +/// \file Kokkos_Pair.hpp +/// \brief Declaration and definition of Kokkos::pair. +/// +/// This header file declares and defines Kokkos::pair and its related +/// nonmember functions. + +#ifndef KOKKOS_PAIR_HPP +#define KOKKOS_PAIR_HPP + +#include +#include + +namespace Kokkos { +/// \struct pair +/// \brief Replacement for std::pair that works on CUDA devices. +/// +/// The instance methods of std::pair, including its constructors, are +/// not marked as __device__ functions. Thus, they cannot be +/// called on a CUDA device, such as an NVIDIA GPU. This struct +/// implements the same interface as std::pair, but can be used on a +/// CUDA device as well as on the host. +template +struct pair +{ + //! The first template parameter of this class. + typedef T1 first_type; + //! The second template parameter of this class. + typedef T2 second_type; + + //! The first element of the pair. + first_type first; + //! The second element of the pair. + second_type second; + + /// \brief Default constructor. + /// + /// This calls the default constructors of T1 and T2. It won't + /// compile if those default constructors are not defined and + /// public. + KOKKOS_FORCEINLINE_FUNCTION + pair() + : first(), second() + {} + + /// \brief Constructor that takes both elements of the pair. + /// + /// This calls the copy constructors of T1 and T2. It won't compile + /// if those copy constructors are not defined and public. + KOKKOS_FORCEINLINE_FUNCTION + pair(first_type const& f, second_type const& s) + : first(f), second(s) + {} + + /// \brief Copy constructor. + /// + /// This calls the copy constructors of T1 and T2. It won't compile + /// if those copy constructors are not defined and public. + template + KOKKOS_FORCEINLINE_FUNCTION + pair( const pair &p) + : first(p.first), second(p.second) + {} + + /// \brief Assignment operator. + /// + /// This calls the assignment operators of T1 and T2. It won't + /// compile if the assignment operators are not defined and public. + template + KOKKOS_FORCEINLINE_FUNCTION + pair & operator=(const pair &p) + { + first = p.first; + second = p.second; + return *this; + } + + // from std::pair + template + pair( const std::pair &p) + : first(p.first), second(p.second) + {} + + /// \brief Return the std::pair version of this object. + /// + /// This is not a device function; you may not call it on a + /// CUDA device. It is meant to be called on the host, if the user + /// wants an std::pair instead of a Kokkos::pair. + /// + /// \note This is not a conversion operator, since defining a + /// conversion operator made the relational operators have + /// ambiguous definitions. + std::pair to_std_pair() const + { return std::make_pair(first,second); } +}; + +template +struct pair +{ + //! The first template parameter of this class. + typedef T1& first_type; + //! The second template parameter of this class. + typedef T2& second_type; + + //! The first element of the pair. + first_type first; + //! The second element of the pair. + second_type second; + + /// \brief Constructor that takes both elements of the pair. + /// + /// This calls the copy constructors of T1 and T2. It won't compile + /// if those copy constructors are not defined and public. + KOKKOS_FORCEINLINE_FUNCTION + pair(first_type f, second_type s) + : first(f), second(s) + {} + + /// \brief Copy constructor. + /// + /// This calls the copy constructors of T1 and T2. It won't compile + /// if those copy constructors are not defined and public. + template + KOKKOS_FORCEINLINE_FUNCTION + pair( const pair &p) + : first(p.first), second(p.second) + {} + + // from std::pair + template + pair( const std::pair &p) + : first(p.first), second(p.second) + {} + + /// \brief Assignment operator. + /// + /// This calls the assignment operators of T1 and T2. It won't + /// compile if the assignment operators are not defined and public. + template + KOKKOS_FORCEINLINE_FUNCTION + pair & operator=(const pair &p) + { + first = p.first; + second = p.second; + return *this; + } + + /// \brief Return the std::pair version of this object. + /// + /// This is not a device function; you may not call it on a + /// CUDA device. It is meant to be called on the host, if the user + /// wants an std::pair instead of a Kokkos::pair. + /// + /// \note This is not a conversion operator, since defining a + /// conversion operator made the relational operators have + /// ambiguous definitions. + std::pair to_std_pair() const + { return std::make_pair(first,second); } +}; + +template +struct pair +{ + //! The first template parameter of this class. + typedef T1 first_type; + //! The second template parameter of this class. + typedef T2& second_type; + + //! The first element of the pair. + first_type first; + //! The second element of the pair. + second_type second; + + /// \brief Constructor that takes both elements of the pair. + /// + /// This calls the copy constructors of T1 and T2. It won't compile + /// if those copy constructors are not defined and public. + KOKKOS_FORCEINLINE_FUNCTION + pair(first_type const& f, second_type s) + : first(f), second(s) + {} + + /// \brief Copy constructor. + /// + /// This calls the copy constructors of T1 and T2. It won't compile + /// if those copy constructors are not defined and public. + template + KOKKOS_FORCEINLINE_FUNCTION + pair( const pair &p) + : first(p.first), second(p.second) + {} + + // from std::pair + template + pair( const std::pair &p) + : first(p.first), second(p.second) + {} + + /// \brief Assignment operator. + /// + /// This calls the assignment operators of T1 and T2. It won't + /// compile if the assignment operators are not defined and public. + template + KOKKOS_FORCEINLINE_FUNCTION + pair & operator=(const pair &p) + { + first = p.first; + second = p.second; + return *this; + } + + /// \brief Return the std::pair version of this object. + /// + /// This is not a device function; you may not call it on a + /// CUDA device. It is meant to be called on the host, if the user + /// wants an std::pair instead of a Kokkos::pair. + /// + /// \note This is not a conversion operator, since defining a + /// conversion operator made the relational operators have + /// ambiguous definitions. + std::pair to_std_pair() const + { return std::make_pair(first,second); } +}; + +template +struct pair +{ + //! The first template parameter of this class. + typedef T1& first_type; + //! The second template parameter of this class. + typedef T2 second_type; + + //! The first element of the pair. + first_type first; + //! The second element of the pair. + second_type second; + + /// \brief Constructor that takes both elements of the pair. + /// + /// This calls the copy constructors of T1 and T2. It won't compile + /// if those copy constructors are not defined and public. + KOKKOS_FORCEINLINE_FUNCTION + pair(first_type f, second_type const& s) + : first(f), second(s) + {} + + /// \brief Copy constructor. + /// + /// This calls the copy constructors of T1 and T2. It won't compile + /// if those copy constructors are not defined and public. + template + KOKKOS_FORCEINLINE_FUNCTION + pair( const pair &p) + : first(p.first), second(p.second) + {} + + // from std::pair + template + pair( const std::pair &p) + : first(p.first), second(p.second) + {} + + /// \brief Assignment operator. + /// + /// This calls the assignment operators of T1 and T2. It won't + /// compile if the assignment operators are not defined and public. + template + KOKKOS_FORCEINLINE_FUNCTION + pair & operator=(const pair &p) + { + first = p.first; + second = p.second; + return *this; + } + + /// \brief Return the std::pair version of this object. + /// + /// This is not a device function; you may not call it on a + /// CUDA device. It is meant to be called on the host, if the user + /// wants an std::pair instead of a Kokkos::pair. + /// + /// \note This is not a conversion operator, since defining a + /// conversion operator made the relational operators have + /// ambiguous definitions. + std::pair to_std_pair() const + { return std::make_pair(first,second); } +}; + +//! Equality operator for Kokkos::pair. +template +KOKKOS_FORCEINLINE_FUNCTION +bool operator== (const pair& lhs, const pair& rhs) +{ return lhs.first==rhs.first && lhs.second==rhs.second; } + +//! Inequality operator for Kokkos::pair. +template +KOKKOS_FORCEINLINE_FUNCTION +bool operator!= (const pair& lhs, const pair& rhs) +{ return !(lhs==rhs); } + +//! Less-than operator for Kokkos::pair. +template +KOKKOS_FORCEINLINE_FUNCTION +bool operator< (const pair& lhs, const pair& rhs) +{ return lhs.first +KOKKOS_FORCEINLINE_FUNCTION +bool operator<= (const pair& lhs, const pair& rhs) +{ return !(rhs +KOKKOS_FORCEINLINE_FUNCTION +bool operator> (const pair& lhs, const pair& rhs) +{ return rhs +KOKKOS_FORCEINLINE_FUNCTION +bool operator>= (const pair& lhs, const pair& rhs) +{ return !(lhs +KOKKOS_FORCEINLINE_FUNCTION +pair make_pair (T1 x, T2 y) +{ return ( pair(x,y) ); } + +/// \brief Return a pair of references to the input arguments. +/// +/// This compares to std::tie (new in C++11). You can use it to +/// assign to two variables at once, from the result of a function +/// that returns a pair. For example (__device__ and +/// __host__ attributes omitted for brevity): +/// \code +/// // Declaration of the function to call. +/// // First return value: operation count. +/// // Second return value: whether all operations succeeded. +/// Kokkos::pair someFunction (); +/// +/// // Code that uses Kokkos::tie. +/// int myFunction () { +/// int count = 0; +/// bool success = false; +/// +/// // This assigns to both count and success. +/// Kokkos::tie (count, success) = someFunction (); +/// +/// if (! success) { +/// // ... Some operation failed; +/// // take corrective action ... +/// } +/// return count; +/// } +/// \endcode +/// +/// The line that uses tie() could have been written like this: +/// \code +/// Kokkos::pair result = someFunction (); +/// count = result.first; +/// success = result.second; +/// \endcode +/// +/// Using tie() saves two lines of code and avoids a copy of each +/// element of the pair. The latter could be significant if one or +/// both elements of the pair are more substantial objects than \c int +/// or \c bool. +template +KOKKOS_FORCEINLINE_FUNCTION +pair tie (T1 & x, T2 & y) +{ return ( pair(x,y) ); } + +// +// Specialization of Kokkos::pair for a \c void second argument. This +// is not actually a "pair"; it only contains one element, the first. +// +template +struct pair +{ + typedef T1 first_type; + typedef void second_type; + + first_type first; + enum { second = 0 }; + + KOKKOS_FORCEINLINE_FUNCTION + pair() + : first() + {} + + KOKKOS_FORCEINLINE_FUNCTION + pair(const first_type & f) + : first(f) + {} + + KOKKOS_FORCEINLINE_FUNCTION + pair(const first_type & f, int) + : first(f) + {} + + template + KOKKOS_FORCEINLINE_FUNCTION + pair( const pair &p) + : first(p.first) + {} + + template + KOKKOS_FORCEINLINE_FUNCTION + pair & operator=(const pair &p) + { + first = p.first; + return *this; + } +}; + +// +// Specialization of relational operators for Kokkos::pair. +// + +template +KOKKOS_FORCEINLINE_FUNCTION +bool operator== (const pair& lhs, const pair& rhs) +{ return lhs.first==rhs.first; } + +template +KOKKOS_FORCEINLINE_FUNCTION +bool operator!= (const pair& lhs, const pair& rhs) +{ return !(lhs==rhs); } + +template +KOKKOS_FORCEINLINE_FUNCTION +bool operator< (const pair& lhs, const pair& rhs) +{ return lhs.first +KOKKOS_FORCEINLINE_FUNCTION +bool operator<= (const pair& lhs, const pair& rhs) +{ return !(rhs +KOKKOS_FORCEINLINE_FUNCTION +bool operator> (const pair& lhs, const pair& rhs) +{ return rhs +KOKKOS_FORCEINLINE_FUNCTION +bool operator>= (const pair& lhs, const pair& rhs) +{ return !(lhs +#include +#include +#include + +#include +#include +#include + +//---------------------------------------------------------------------------- +//---------------------------------------------------------------------------- + +namespace Kokkos { +namespace Impl { + +//---------------------------------------------------------------------------- +/** \brief Given a Functor and Execution Policy query an execution space. + * + * if the Policy has an execution space use that + * else if the Functor has an execution_space use that + * else if the Functor has a device_type use that for backward compatibility + * else use the default + */ +template< class Functor + , class Policy + , class EnableFunctor = void + , class EnablePolicy = void + > +struct FunctorPolicyExecutionSpace { + typedef Kokkos::DefaultExecutionSpace execution_space ; +}; + +template< class Functor , class Policy > +struct FunctorPolicyExecutionSpace + < Functor , Policy + , typename enable_if_type< typename Functor::device_type >::type + , typename enable_if_type< typename Policy ::execution_space >::type + > +{ + typedef typename Policy ::execution_space execution_space ; +}; + +template< class Functor , class Policy > +struct FunctorPolicyExecutionSpace + < Functor , Policy + , typename enable_if_type< typename Functor::execution_space >::type + , typename enable_if_type< typename Policy ::execution_space >::type + > +{ + typedef typename Policy ::execution_space execution_space ; +}; + +template< class Functor , class Policy , class EnableFunctor > +struct FunctorPolicyExecutionSpace + < Functor , Policy + , EnableFunctor + , typename enable_if_type< typename Policy::execution_space >::type + > +{ + typedef typename Policy ::execution_space execution_space ; +}; + +template< class Functor , class Policy , class EnablePolicy > +struct FunctorPolicyExecutionSpace + < Functor , Policy + , typename enable_if_type< typename Functor::device_type >::type + , EnablePolicy + > +{ + typedef typename Functor::device_type execution_space ; +}; + +template< class Functor , class Policy , class EnablePolicy > +struct FunctorPolicyExecutionSpace + < Functor , Policy + , typename enable_if_type< typename Functor::execution_space >::type + , EnablePolicy + > +{ + typedef typename Functor::execution_space execution_space ; +}; + +//---------------------------------------------------------------------------- +/// \class ParallelFor +/// \brief Implementation of the ParallelFor operator that has a +/// partial specialization for the device. +/// +/// This is an implementation detail of parallel_for. Users should +/// skip this and go directly to the nonmember function parallel_for. +template< class FunctorType , class ExecPolicy > class ParallelFor ; + +/// \class ParallelReduce +/// \brief Implementation detail of parallel_reduce. +/// +/// This is an implementation detail of parallel_reduce. Users should +/// skip this and go directly to the nonmember function parallel_reduce. +template< class FunctorType , class ExecPolicy > class ParallelReduce ; + +/// \class ParallelScan +/// \brief Implementation detail of parallel_scan. +/// +/// This is an implementation detail of parallel_scan. Users should +/// skip this and go directly to the documentation of the nonmember +/// template function Kokkos::parallel_scan. +template< class FunctorType , class ExecPolicy > class ParallelScan ; + +} // namespace Impl +} // namespace Kokkos + +//---------------------------------------------------------------------------- +//---------------------------------------------------------------------------- + +namespace Kokkos { + +/** \brief Execute \c functor in parallel according to the execution \c policy. + * + * A "functor" is a class containing the function to execute in parallel, + * data needed for that execution, and an optional \c execution_space + * typedef. Here is an example functor for parallel_for: + * + * \code + * class FunctorType { + * public: + * typedef ... execution_space ; + * void operator() ( WorkType iwork ) const ; + * }; + * \endcode + * + * In the above example, \c WorkType is any integer type for which a + * valid conversion from \c size_t to \c IntType exists. Its + * operator() method defines the operation to parallelize, + * over the range of integer indices iwork=[0,work_count-1]. + * This compares to a single iteration \c iwork of a \c for loop. + * If \c execution_space is not defined DefaultExecutionSpace will be used. + */ +template< class ExecPolicy , class FunctorType > +inline +void parallel_for( const ExecPolicy & policy + , const FunctorType & functor + , typename Impl::enable_if< ! Impl::is_integral< ExecPolicy >::value >::type * = 0 + ) +{ + (void) Impl::ParallelFor< FunctorType , ExecPolicy >( functor , policy ); +} + +template< class FunctorType > +inline +void parallel_for( const size_t work_count , + const FunctorType & functor ) +{ + typedef typename + Impl::FunctorPolicyExecutionSpace< FunctorType , void >::execution_space + execution_space ; + typedef RangePolicy< execution_space > policy ; + (void) Impl::ParallelFor< FunctorType , policy >( functor , policy(0,work_count) ); +} + +//---------------------------------------------------------------------------- +//---------------------------------------------------------------------------- + +/** \brief Parallel reduction + * + * Example of a parallel_reduce functor for a POD (plain old data) value type: + * \code + * class FunctorType { // For POD value type + * public: + * typedef ... execution_space ; + * typedef value_type ; + * void operator()( iwork , & update ) const ; + * void init( & update ) const ; + * void join( volatile & update , + * volatile const & input ) const ; + * + * typedef true_type has_final ; + * void final( & update ) const ; + * }; + * \endcode + * + * Example of a parallel_reduce functor for an array of POD (plain old data) values: + * \code + * class FunctorType { // For array of POD value + * public: + * typedef ... execution_space ; + * typedef value_type[] ; + * void operator()( , update[] ) const ; + * void init( update[] ) const ; + * void join( volatile update[] , + * volatile const input[] ) const ; + * + * typedef true_type has_final ; + * void final( update[] ) const ; + * }; + * \endcode + */ +template< class ExecPolicy , class FunctorType > +inline +void parallel_reduce( const ExecPolicy & policy + , const FunctorType & functor + , typename Impl::enable_if< ! Impl::is_integral< ExecPolicy >::value >::type * = 0 + ) +{ + (void) Impl::ParallelReduce< FunctorType , ExecPolicy >( functor , policy ); +} + +// integral range policy +template< class FunctorType > +inline +void parallel_reduce( const size_t work_count + , const FunctorType & functor + ) +{ + typedef typename + Impl::FunctorPolicyExecutionSpace< FunctorType , void >::execution_space + execution_space ; + + typedef RangePolicy< execution_space > policy ; + + typedef Kokkos::Impl::FunctorValueTraits< FunctorType , void > ValueTraits ; + + typedef typename Kokkos::Impl::if_c< (ValueTraits::StaticValueSize != 0) + , typename ValueTraits::value_type + , typename ValueTraits::pointer_type + >::type value_type ; + + Kokkos::View< value_type + , HostSpace + , Kokkos::MemoryUnmanaged + > + result_view ; + + (void) Impl::ParallelReduce< FunctorType , policy >( functor , policy(0,work_count) , result_view ); +} + +// general policy and view ouput +template< class ExecPolicy , class FunctorType , class ViewType > +inline +void parallel_reduce( const ExecPolicy & policy + , const FunctorType & functor + , const ViewType & result_view + , typename Impl::enable_if< + ( Impl::is_view::value && ! Impl::is_integral< ExecPolicy >::value + )>::type * = 0 ) +{ + (void) Impl::ParallelReduce< FunctorType, ExecPolicy >( functor , policy , result_view ); +} + +// general policy and pod or array of pod output +template< class ExecPolicy , class FunctorType > +inline +void parallel_reduce( const ExecPolicy & policy + , const FunctorType & functor + , typename Impl::enable_if< + ( ! Impl::is_integral< ExecPolicy >::value ) + , typename Kokkos::Impl::FunctorValueTraits< FunctorType , typename ExecPolicy::work_tag >::reference_type + >::type result_ref ) +{ + typedef Kokkos::Impl::FunctorValueTraits< FunctorType , typename ExecPolicy::work_tag > ValueTraits ; + typedef Kokkos::Impl::FunctorValueOps< FunctorType , typename ExecPolicy::work_tag > ValueOps ; + + // Wrap the result output request in a view to inform the implementation + // of the type and memory space. + + typedef typename Kokkos::Impl::if_c< (ValueTraits::StaticValueSize != 0) + , typename ValueTraits::value_type + , typename ValueTraits::pointer_type + >::type value_type ; + + Kokkos::View< value_type + , HostSpace + , Kokkos::MemoryUnmanaged + > + result_view( ValueOps::pointer( result_ref ) + , ValueTraits::value_count( functor ) + ); + + (void) Impl::ParallelReduce< FunctorType, ExecPolicy >( functor , policy , result_view ); +} + +// integral range policy and view ouput +template< class FunctorType , class ViewType > +inline +void parallel_reduce( const size_t work_count + , const FunctorType & functor + , const ViewType & result_view + , typename Impl::enable_if<( Impl::is_view::value )>::type * = 0 ) +{ + typedef typename + Impl::FunctorPolicyExecutionSpace< FunctorType , void >::execution_space + execution_space ; + + typedef RangePolicy< execution_space > ExecPolicy ; + + (void) Impl::ParallelReduce< FunctorType, ExecPolicy >( functor , ExecPolicy(0,work_count) , result_view ); +} + +// integral range policy and pod or array of pod output +template< class FunctorType > +inline +void parallel_reduce( const size_t work_count , + const FunctorType & functor , + typename Kokkos::Impl::FunctorValueTraits< FunctorType , void >::reference_type result ) +{ + typedef Kokkos::Impl::FunctorValueTraits< FunctorType , void > ValueTraits ; + typedef Kokkos::Impl::FunctorValueOps< FunctorType , void > ValueOps ; + + typedef typename + Kokkos::Impl::FunctorPolicyExecutionSpace< FunctorType , void >::execution_space + execution_space ; + + typedef Kokkos::RangePolicy< execution_space > policy ; + + // Wrap the result output request in a view to inform the implementation + // of the type and memory space. + + typedef typename Kokkos::Impl::if_c< (ValueTraits::StaticValueSize != 0) + , typename ValueTraits::value_type + , typename ValueTraits::pointer_type + >::type value_type ; + + Kokkos::View< value_type + , HostSpace + , Kokkos::MemoryUnmanaged + > + result_view( ValueOps::pointer( result ) + , ValueTraits::value_count( functor ) + ); + + (void) Impl::ParallelReduce< FunctorType , policy >( functor , policy(0,work_count) , result_view ); +} + +} // namespace Kokkos + +//---------------------------------------------------------------------------- +//---------------------------------------------------------------------------- + +namespace Kokkos { + +/// \fn parallel_scan +/// \tparam ExecutionPolicy The execution policy type. +/// \tparam FunctorType The scan functor type. +/// +/// \param policy [in] The execution policy. +/// \param functor [in] The scan functor. +/// +/// This function implements a parallel scan pattern. The scan can +/// be either inclusive or exclusive, depending on how you implement +/// the scan functor. +/// +/// A scan functor looks almost exactly like a reduce functor, except +/// that its operator() takes a third \c bool argument, \c final_pass, +/// which indicates whether this is the last pass of the scan +/// operation. We will show below how to use the \c final_pass +/// argument to control whether the scan is inclusive or exclusive. +/// +/// Here is the minimum required interface of a scan functor for a POD +/// (plain old data) value type \c PodType. That is, the result is a +/// View of zero or more PodType. It is also possible for the result +/// to be an array of (same-sized) arrays of PodType, but we do not +/// show the required interface for that here. +/// \code +/// template< class ExecPolicy , class FunctorType > +/// class ScanFunctor { +/// public: +/// // The Kokkos device type +/// typedef ... execution_space; +/// // Type of an entry of the array containing the result; +/// // also the type of each of the entries combined using +/// // operator() or join(). +/// typedef PodType value_type; +/// +/// void operator () (const ExecPolicy::member_type & i, value_type& update, const bool final_pass) const; +/// void init (value_type& update) const; +/// void join (volatile value_type& update, volatile const value_type& input) const +/// }; +/// \endcode +/// +/// Here is an example of a functor which computes an inclusive plus-scan +/// of an array of \c int, in place. If given an array [1, 2, 3, 4], this +/// scan will overwrite that array with [1, 3, 6, 10]. +/// +/// \code +/// template +/// class InclScanFunctor { +/// public: +/// typedef SpaceType execution_space; +/// typedef int value_type; +/// typedef typename SpaceType::size_type size_type; +/// +/// InclScanFunctor( Kokkos::View x +/// , Kokkos::View y ) : m_x(x), m_y(y) {} +/// +/// void operator () (const size_type i, value_type& update, const bool final_pass) const { +/// update += m_x(i); +/// if (final_pass) { +/// m_y(i) = update; +/// } +/// } +/// void init (value_type& update) const { +/// update = 0; +/// } +/// void join (volatile value_type& update, volatile const value_type& input) const { +/// update += input; +/// } +/// +/// private: +/// Kokkos::View m_x; +/// Kokkos::View m_y; +/// }; +/// \endcode +/// +/// Here is an example of a functor which computes an exclusive +/// scan of an array of \c int, in place. In operator(), note both +/// that the final_pass test and the update have switched places, and +/// the use of a temporary. If given an array [1, 2, 3, 4], this scan +/// will overwrite that array with [0, 1, 3, 6]. +/// +/// \code +/// template +/// class ExclScanFunctor { +/// public: +/// typedef SpaceType execution_space; +/// typedef int value_type; +/// typedef typename SpaceType::size_type size_type; +/// +/// ExclScanFunctor (Kokkos::View x) : x_ (x) {} +/// +/// void operator () (const size_type i, value_type& update, const bool final_pass) const { +/// const value_type x_i = x_(i); +/// if (final_pass) { +/// x_(i) = update; +/// } +/// update += x_i; +/// } +/// void init (value_type& update) const { +/// update = 0; +/// } +/// void join (volatile value_type& update, volatile const value_type& input) const { +/// update += input; +/// } +/// +/// private: +/// Kokkos::View x_; +/// }; +/// \endcode +/// +/// Here is an example of a functor which builds on the above +/// exclusive scan example, to compute an offsets array from a +/// population count array, in place. We assume that the pop count +/// array has an extra entry at the end to store the final count. If +/// given an array [1, 2, 3, 4, 0], this scan will overwrite that +/// array with [0, 1, 3, 6, 10]. +/// +/// \code +/// template +/// class OffsetScanFunctor { +/// public: +/// typedef SpaceType execution_space; +/// typedef int value_type; +/// typedef typename SpaceType::size_type size_type; +/// +/// // lastIndex_ is the last valid index (zero-based) of x. +/// // If x has length zero, then lastIndex_ won't be used anyway. +/// OffsetScanFunctor( Kokkos::View x +/// , Kokkos::View y ) +/// : m_x(x), m_y(y), last_index_ (x.dimension_0 () == 0 ? 0 : x.dimension_0 () - 1) +/// {} +/// +/// void operator () (const size_type i, int& update, const bool final_pass) const { +/// if (final_pass) { +/// m_y(i) = update; +/// } +/// update += m_x(i); +/// // The last entry of m_y gets the final sum. +/// if (final_pass && i == last_index_) { +/// m_y(i+1) = update; +/// } +/// } +/// void init (value_type& update) const { +/// update = 0; +/// } +/// void join (volatile value_type& update, volatile const value_type& input) const { +/// update += input; +/// } +/// +/// private: +/// Kokkos::View m_x; +/// Kokkos::View m_y; +/// const size_type last_index_; +/// }; +/// \endcode +/// +template< class ExecutionPolicy , class FunctorType > +inline +void parallel_scan( const ExecutionPolicy & policy + , const FunctorType & functor + , typename Impl::enable_if< ! Impl::is_integral< ExecutionPolicy >::value >::type * = 0 + ) +{ + Impl::ParallelScan< FunctorType , ExecutionPolicy > scan( functor , policy ); +} + +template< class FunctorType > +inline +void parallel_scan( const size_t work_count , + const FunctorType & functor ) +{ + typedef typename + Kokkos::Impl::FunctorPolicyExecutionSpace< FunctorType , void >::execution_space + execution_space ; + + typedef Kokkos::RangePolicy< execution_space > policy ; + + (void) Impl::ParallelScan< FunctorType , policy >( functor , policy(0,work_count) ); +} + +} // namespace Kokkos + +//---------------------------------------------------------------------------- +//---------------------------------------------------------------------------- + +namespace Kokkos { +namespace Impl { + +template< class FunctorType , class Enable = void > +struct FunctorTeamShmemSize +{ + static inline size_t value( const FunctorType & , int ) { return 0 ; } +}; + +template< class FunctorType > +struct FunctorTeamShmemSize< FunctorType , typename enable_if< sizeof( & FunctorType::team_shmem_size ) >::type > +{ + static inline size_t value( const FunctorType & f , int team_size ) { return f.team_shmem_size( team_size ) ; } +}; + +template< class FunctorType > +struct FunctorTeamShmemSize< FunctorType , typename enable_if< sizeof( & FunctorType::shmem_size ) >::type > +{ + static inline size_t value( const FunctorType & f , int team_size ) { return f.shmem_size( team_size ) ; } +}; + +} // namespace Impl +} // namespace Kokkos + +//---------------------------------------------------------------------------- +//---------------------------------------------------------------------------- + +#endif /* KOKKOS_PARALLEL_HPP */ + diff --git a/lib/kokkos/core/src/Kokkos_Qthread.hpp b/lib/kokkos/core/src/Kokkos_Qthread.hpp new file mode 100755 index 0000000000..cc6f0f844b --- /dev/null +++ b/lib/kokkos/core/src/Kokkos_Qthread.hpp @@ -0,0 +1,165 @@ +/* +//@HEADER +// ************************************************************************ +// +// Kokkos: Manycore Performance-Portable Multidimensional Arrays +// Copyright (2012) Sandia Corporation +// +// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +// the U.S. Government retains certain rights in this software. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// 1. Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// 3. Neither the name of the Corporation nor the names of the +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Questions? Contact H. Carter Edwards (hcedwar@sandia.gov) +// +// ************************************************************************ +//@HEADER +*/ + +#ifndef KOKKOS_QTHREAD_HPP +#define KOKKOS_QTHREAD_HPP + +#include +#include +#include +#include +#include +#include +#include +#include + +/*--------------------------------------------------------------------------*/ + +namespace Kokkos { +namespace Impl { +class QthreadExec ; +} // namespace Impl +} // namespace Kokkos + +/*--------------------------------------------------------------------------*/ + +namespace Kokkos { + +/** \brief Execution space supported by Qthread */ +class Qthread { +public: + //! \name Type declarations that all Kokkos devices must provide. + //@{ + + //! Tag this class as an execution space + typedef Qthread execution_space ; + typedef Kokkos::HostSpace memory_space ; + typedef Kokkos::LayoutRight array_layout ; + typedef memory_space::size_type size_type ; + + typedef ScratchMemorySpace< Qthread > scratch_memory_space ; + + //! For backward compatibility: + typedef Qthread device_type ; + + //@} + /*------------------------------------------------------------------------*/ + + /** \brief Initialization will construct one or more instances */ + static Qthread & instance( int = 0 ); + + /** \brief Set the execution space to a "sleep" state. + * + * This function sets the "sleep" state in which it is not ready for work. + * This may consume less resources than in an "ready" state, + * but it may also take time to transition to the "ready" state. + * + * \return True if enters or is in the "sleep" state. + * False if functions are currently executing. + */ + bool sleep(); + + /** \brief Wake from the sleep state. + * + * \return True if enters or is in the "ready" state. + * False if functions are currently executing. + */ + static bool wake(); + + /** \brief Wait until all dispatched functions to complete. + * + * The parallel_for or parallel_reduce dispatch of a functor may + * return asynchronously, before the functor completes. This + * method does not return until all dispatched functors on this + * device have completed. + */ + static void fence(); + + /*------------------------------------------------------------------------*/ + + static void initialize( int thread_count ); + static void finalize(); + + /** \brief Print configuration information to the given output stream. */ + static void print_configuration( std::ostream & , const bool detail = false ); + + int shepherd_size() const ; + int shepherd_worker_size() const ; +}; + +/*--------------------------------------------------------------------------*/ + +} // namespace Kokkos + +/*--------------------------------------------------------------------------*/ +/*--------------------------------------------------------------------------*/ + +namespace Kokkos { +namespace Impl { + +template<> +struct VerifyExecutionCanAccessMemorySpace + < Kokkos::Qthread::memory_space + , Kokkos::Qthread::scratch_memory_space + > +{ + enum { value = true }; + inline static void verify( void ) { } + inline static void verify( const void * ) { } +}; + +} // namespace Impl +} // namespace Kokkos + +/*--------------------------------------------------------------------------*/ +/*--------------------------------------------------------------------------*/ + +#include +#include +#include + +#endif /* #define KOKKOS_QTHREAD_HPP */ + +//---------------------------------------------------------------------------- +//---------------------------------------------------------------------------- + diff --git a/lib/kokkos/core/src/Kokkos_ScratchSpace.hpp b/lib/kokkos/core/src/Kokkos_ScratchSpace.hpp new file mode 100755 index 0000000000..56b954d9bf --- /dev/null +++ b/lib/kokkos/core/src/Kokkos_ScratchSpace.hpp @@ -0,0 +1,115 @@ +/* +//@HEADER +// ************************************************************************ +// +// Kokkos: Manycore Performance-Portable Multidimensional Arrays +// Copyright (2012) Sandia Corporation +// +// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +// the U.S. Government retains certain rights in this software. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// 1. Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// 3. Neither the name of the Corporation nor the names of the +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Questions? Contact H. Carter Edwards (hcedwar@sandia.gov) +// +// ************************************************************************ +//@HEADER +*/ + +#ifndef KOKKOS_SCRATCHSPACE_HPP +#define KOKKOS_SCRATCHSPACE_HPP + +#include +#include +#include + +/*--------------------------------------------------------------------------*/ + +namespace Kokkos { + +/** \brief Scratch memory space associated with an execution space. + * + */ +template< class ExecSpace > +class ScratchMemorySpace { +public: + + // Alignment of memory chunks returned by 'get' + // must be a power of two + enum { ALIGN = 8 }; + +private: + + mutable char * m_iter ; + char * m_end ; + + ScratchMemorySpace(); + ScratchMemorySpace & operator = ( const ScratchMemorySpace & ); + + enum { MASK = ALIGN - 1 }; // Alignment used by View::shmem_size + +public: + + //! Tag this class as a memory space + typedef ScratchMemorySpace memory_space ; + typedef ExecSpace execution_space ; + typedef typename ExecSpace::array_layout array_layout ; + typedef typename ExecSpace::size_type size_type ; + + template< typename IntType > + KOKKOS_INLINE_FUNCTION static + IntType align( const IntType & size ) + { return ( size + MASK ) & ~MASK ; } + + template< typename IntType > + KOKKOS_INLINE_FUNCTION + void* get_shmem (const IntType& size) const { + void* tmp = m_iter ; + if (m_end < (m_iter += align (size))) { + m_iter -= align (size); // put it back like it was + printf ("ScratchMemorySpace<...>::get_shmem: Failed to allocate %ld byte(s); remaining capacity is %ld byte(s)\n", long(size), long(m_end-m_iter)); + tmp = 0; + } + return tmp; + } + + template< typename IntType > + KOKKOS_INLINE_FUNCTION + ScratchMemorySpace( void * ptr , const IntType & size ) + : m_iter( (char *) ptr ) + , m_end( m_iter + size ) + {} +}; + +} // namespace Kokkos + +#endif /* #ifndef KOKKOS_SCRATCHSPACE_HPP */ + +//---------------------------------------------------------------------------- +//---------------------------------------------------------------------------- + diff --git a/lib/kokkos/core/src/Kokkos_Serial.hpp b/lib/kokkos/core/src/Kokkos_Serial.hpp new file mode 100755 index 0000000000..e9495724f1 --- /dev/null +++ b/lib/kokkos/core/src/Kokkos_Serial.hpp @@ -0,0 +1,879 @@ +/* +//@HEADER +// ************************************************************************ +// +// Kokkos: Manycore Performance-Portable Multidimensional Arrays +// Copyright (2012) Sandia Corporation +// +// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +// the U.S. Government retains certain rights in this software. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// 1. Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// 3. Neither the name of the Corporation nor the names of the +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Questions? Contact H. Carter Edwards (hcedwar@sandia.gov) +// +// ************************************************************************ +//@HEADER +*/ + +/// \file Kokkos_Serial.hpp +/// \brief Declaration and definition of Kokkos::Serial device. + +#ifndef KOKKOS_SERIAL_HPP +#define KOKKOS_SERIAL_HPP + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#if defined( KOKKOS_HAVE_SERIAL ) + +namespace Kokkos { + +/// \class Serial +/// \brief Kokkos device for non-parallel execution +/// +/// A "device" represents a parallel execution model. It tells Kokkos +/// how to parallelize the execution of kernels in a parallel_for or +/// parallel_reduce. For example, the Threads device uses Pthreads or +/// C++11 threads on a CPU, the OpenMP device uses the OpenMP language +/// extensions, and the Cuda device uses NVIDIA's CUDA programming +/// model. The Serial device executes "parallel" kernels +/// sequentially. This is useful if you really do not want to use +/// threads, or if you want to explore different combinations of MPI +/// and shared-memory parallel programming models. +class Serial { +public: + //! \name Type declarations that all Kokkos devices must provide. + //@{ + + //! Tag this class as an execution space: + typedef Serial execution_space ; + //! The size_type typedef best suited for this device. + typedef HostSpace::size_type size_type ; + //! This device's preferred memory space. + typedef HostSpace memory_space ; + //! This device's preferred array layout. + typedef LayoutRight array_layout ; + + /// \brief Scratch memory space + typedef ScratchMemorySpace< Kokkos::Serial > scratch_memory_space ; + + //! For backward compatibility: + typedef Serial device_type ; + + //@} + + /// \brief True if and only if this method is being called in a + /// thread-parallel function. + /// + /// For the Serial device, this method always returns false, + /// because parallel_for or parallel_reduce with the Serial device + /// always execute sequentially. + inline static int in_parallel() { return false ; } + + /** \brief Set the device in a "sleep" state. + * + * This function sets the device in a "sleep" state in which it is + * not ready for work. This may consume less resources than if the + * device were in an "awake" state, but it may also take time to + * bring the device from a sleep state to be ready for work. + * + * \return True if the device is in the "sleep" state, else false if + * the device is actively working and could not enter the "sleep" + * state. + */ + static bool sleep(); + + /// \brief Wake the device from the 'sleep' state so it is ready for work. + /// + /// \return True if the device is in the "ready" state, else "false" + /// if the device is actively working (which also means that it's + /// awake). + static bool wake(); + + /// \brief Wait until all dispatched functors complete. + /// + /// The parallel_for or parallel_reduce dispatch of a functor may + /// return asynchronously, before the functor completes. This + /// method does not return until all dispatched functors on this + /// device have completed. + static void fence() {} + + static void initialize( unsigned threads_count = 1 , + unsigned use_numa_count = 0 , + unsigned use_cores_per_numa = 0 , + bool allow_asynchronous_threadpool = false) { + (void) threads_count; + (void) use_numa_count; + (void) use_cores_per_numa; + (void) allow_asynchronous_threadpool; + } + + static int is_initialized() { return 1 ; } + + //! Free any resources being consumed by the device. + static void finalize() {} + + //! Print configuration information to the given output stream. + static void print_configuration( std::ostream & , const bool detail = false ); + + //-------------------------------------------------------------------------- + + inline static int thread_pool_size( int = 0 ) { return 1 ; } + KOKKOS_INLINE_FUNCTION static int thread_pool_rank() { return 0 ; } + + //-------------------------------------------------------------------------- + + KOKKOS_INLINE_FUNCTION static unsigned hardware_thread_id() { return thread_pool_rank(); } + inline static unsigned max_hardware_threads() { return thread_pool_size(0); } + + //-------------------------------------------------------------------------- + + static void * scratch_memory_resize( unsigned reduce_size , unsigned shared_size ); + + //-------------------------------------------------------------------------- +}; + +} // namespace Kokkos + +/*--------------------------------------------------------------------------*/ +/*--------------------------------------------------------------------------*/ + +namespace Kokkos { +namespace Impl { + +template<> +struct VerifyExecutionCanAccessMemorySpace + < Kokkos::Serial::memory_space + , Kokkos::Serial::scratch_memory_space + > +{ + enum { value = true }; + inline static void verify( void ) { } + inline static void verify( const void * ) { } +}; + +namespace SerialImpl { + +struct Sentinel { + + void * m_scratch ; + unsigned m_reduce_end ; + unsigned m_shared_end ; + + Sentinel(); + ~Sentinel(); + static Sentinel & singleton(); +}; + +inline +unsigned align( unsigned n ); +} +} // namespace Impl +} // namespace Kokkos + +/*--------------------------------------------------------------------------*/ +/*--------------------------------------------------------------------------*/ + +namespace Kokkos { +namespace Impl { + +class SerialTeamMember { +private: + typedef Kokkos::ScratchMemorySpace< Kokkos::Serial > scratch_memory_space ; + const scratch_memory_space m_space ; + const int m_league_rank ; + const int m_league_size ; + + SerialTeamMember & operator = ( const SerialTeamMember & ); + +public: + + KOKKOS_INLINE_FUNCTION + const scratch_memory_space & team_shmem() const { return m_space ; } + + KOKKOS_INLINE_FUNCTION int league_rank() const { return m_league_rank ; } + KOKKOS_INLINE_FUNCTION int league_size() const { return m_league_size ; } + KOKKOS_INLINE_FUNCTION int team_rank() const { return 0 ; } + KOKKOS_INLINE_FUNCTION int team_size() const { return 1 ; } + + KOKKOS_INLINE_FUNCTION void team_barrier() const {} + + template + KOKKOS_INLINE_FUNCTION + void team_broadcast(const ValueType& , const int& ) const {} + + template< class ValueType, class JoinOp > + KOKKOS_INLINE_FUNCTION + ValueType team_reduce( const ValueType & value + , const JoinOp & ) const + { + return value ; + } + + /** \brief Intra-team exclusive prefix sum with team_rank() ordering + * with intra-team non-deterministic ordering accumulation. + * + * The global inter-team accumulation value will, at the end of the + * league's parallel execution, be the scan's total. + * Parallel execution ordering of the league's teams is non-deterministic. + * As such the base value for each team's scan operation is similarly + * non-deterministic. + */ + template< typename Type > + KOKKOS_INLINE_FUNCTION Type team_scan( const Type & value , Type * const global_accum ) const + { + const Type tmp = global_accum ? *global_accum : Type(0) ; + if ( global_accum ) { *global_accum += value ; } + return tmp ; + } + + /** \brief Intra-team exclusive prefix sum with team_rank() ordering. + * + * The highest rank thread can compute the reduction total as + * reduction_total = dev.team_scan( value ) + value ; + */ + template< typename Type > + KOKKOS_INLINE_FUNCTION Type team_scan( const Type & ) const + { return Type(0); } + +#ifdef KOKKOS_HAVE_CXX11 + + /** \brief Executes op(iType i) for each i=0..N-1. + * + * This functionality requires C++11 support.*/ + template< typename iType, class Operation> + KOKKOS_INLINE_FUNCTION void team_par_for(const iType n, const Operation & op) const { + for(int i=0; i + * < WorkArgTag , Impl::enable_if< Impl::is_same< Kokkos::Serial , Kokkos::DefaultExecutionSpace >::value >::type > + * + */ +template< class Arg0 , class Arg1 > +class TeamPolicy< Arg0 , Arg1 , Kokkos::Serial > +{ +private: + + const int m_league_size ; + +public: + + //! Tag this class as a kokkos execution policy + typedef TeamPolicy execution_policy ; + + //! Execution space of this execution policy: + typedef Kokkos::Serial execution_space ; + + typedef typename + Impl::if_c< ! Impl::is_same< Kokkos::Serial , Arg0 >::value , Arg0 , Arg1 >::type + work_tag ; + + //---------------------------------------- + + template< class FunctorType > + static + int team_size_max( const FunctorType & ) { return 1 ; } + + template< class FunctorType > + static + int team_size_recommended( const FunctorType & ) { return 1 ; } + + //---------------------------------------- + + inline int team_size() const { return 1 ; } + inline int league_size() const { return m_league_size ; } + + /** \brief Specify league size, request team size */ + TeamPolicy( execution_space & , int league_size_request , int /* team_size_request */ , int vector_length_request = 1 ) + : m_league_size( league_size_request ) + { (void) vector_length_request; } + + TeamPolicy( int league_size_request , int /* team_size_request */ , int vector_length_request = 1 ) + : m_league_size( league_size_request ) + { (void) vector_length_request; } + + typedef Impl::SerialTeamMember member_type ; +}; + +} /* namespace Kokkos */ + +/*--------------------------------------------------------------------------*/ +/*--------------------------------------------------------------------------*/ + +/*--------------------------------------------------------------------------*/ +/*--------------------------------------------------------------------------*/ + +namespace Kokkos { +namespace Impl { + +template< class FunctorType , class Arg0 , class Arg1 , class Arg2 > +class ParallelFor< FunctorType , Kokkos::RangePolicy< Arg0 , Arg1 , Arg2 , Kokkos::Serial > > +{ +private: + + typedef Kokkos::RangePolicy< Arg0 , Arg1 , Arg2 , Kokkos::Serial > Policy ; + +public: + // work tag is void + template< class PType > + inline + ParallelFor( typename Impl::enable_if< + ( Impl::is_same< PType , Policy >::value && + Impl::is_same< typename PType::work_tag , void >::value + ), const FunctorType & >::type functor + , const PType & policy ) + { + const typename PType::member_type e = policy.end(); + for ( typename PType::member_type i = policy.begin() ; i < e ; ++i ) { + functor( i ); + } + } + + // work tag is non-void + template< class PType > + inline + ParallelFor( typename Impl::enable_if< + ( Impl::is_same< PType , Policy >::value && + ! Impl::is_same< typename PType::work_tag , void >::value + ), const FunctorType & >::type functor + , const PType & policy ) + { + const typename PType::member_type e = policy.end(); + for ( typename PType::member_type i = policy.begin() ; i < e ; ++i ) { + functor( typename PType::work_tag() , i ); + } + } +}; + +template< class FunctorType , class Arg0 , class Arg1 , class Arg2 > +class ParallelReduce< FunctorType , Kokkos::RangePolicy< Arg0 , Arg1 , Arg2 , Kokkos::Serial > > +{ +public: + typedef Kokkos::RangePolicy< Arg0 , Arg1 , Arg2 , Kokkos::Serial > Policy ; + typedef typename Policy::work_tag WorkTag ; + typedef Kokkos::Impl::FunctorValueTraits< FunctorType , WorkTag > ValueTraits ; + typedef Kokkos::Impl::FunctorValueInit< FunctorType , WorkTag > ValueInit ; + + typedef typename ValueTraits::pointer_type pointer_type ; + typedef typename ValueTraits::reference_type reference_type ; + + // Work tag is void + template< class ViewType , class PType > + ParallelReduce( typename Impl::enable_if< + ( Impl::is_view< ViewType >::value && + Impl::is_same< typename ViewType::memory_space , HostSpace >::value && + Impl::is_same< PType , Policy >::value && + Impl::is_same< typename PType::work_tag , void >::value + ), const FunctorType & >::type functor + , const PType & policy + , const ViewType & result + ) + { + pointer_type result_ptr = result.ptr_on_device(); + + if ( ! result_ptr ) { + result_ptr = (pointer_type) + Kokkos::Serial::scratch_memory_resize( ValueTraits::value_size( functor ) , 0 ); + } + + reference_type update = ValueInit::init( functor , result_ptr ); + + const typename PType::member_type e = policy.end(); + for ( typename PType::member_type i = policy.begin() ; i < e ; ++i ) { + functor( i , update ); + } + + Kokkos::Impl::FunctorFinal< FunctorType , WorkTag >::final( functor , result_ptr ); + } + + // Work tag is non-void + template< class ViewType , class PType > + ParallelReduce( typename Impl::enable_if< + ( Impl::is_view< ViewType >::value && + Impl::is_same< typename ViewType::memory_space , HostSpace >::value && + Impl::is_same< PType , Policy >::value && + ! Impl::is_same< typename PType::work_tag , void >::value + ), const FunctorType & >::type functor + , const PType & policy + , const ViewType & result + ) + { + pointer_type result_ptr = result.ptr_on_device(); + + if ( ! result_ptr ) { + result_ptr = (pointer_type) + Kokkos::Serial::scratch_memory_resize( ValueTraits::value_size( functor ) , 0 ); + } + + typename ValueTraits::reference_type update = ValueInit::init( functor , result_ptr ); + + const typename PType::member_type e = policy.end(); + for ( typename PType::member_type i = policy.begin() ; i < e ; ++i ) { + functor( typename PType::work_tag() , i , update ); + } + + Kokkos::Impl::FunctorFinal< FunctorType , WorkTag >::final( functor , result_ptr ); + } +}; + +template< class FunctorType , class Arg0 , class Arg1 , class Arg2 > +class ParallelScan< FunctorType , Kokkos::RangePolicy< Arg0 , Arg1 , Arg2 , Kokkos::Serial > > +{ +private: + + typedef Kokkos::RangePolicy< Arg0 , Arg1 , Arg2 , Kokkos::Serial > Policy ; + + typedef Kokkos::Impl::FunctorValueTraits< FunctorType , typename Policy::work_tag > ValueTraits ; + typedef Kokkos::Impl::FunctorValueInit< FunctorType , typename Policy::work_tag > ValueInit ; + +public: + + typedef typename ValueTraits::pointer_type pointer_type ; + typedef typename ValueTraits::reference_type reference_type ; + + // work tag is void + template< class PType > + inline + ParallelScan( typename Impl::enable_if< + ( Impl::is_same< PType , Policy >::value && + Impl::is_same< typename PType::work_tag , void >::value + ), const FunctorType & >::type functor + , const PType & policy ) + { + pointer_type result_ptr = (pointer_type) + Kokkos::Serial::scratch_memory_resize( ValueTraits::value_size( functor ) , 0 ); + + reference_type update = ValueInit::init( functor , result_ptr ); + + const typename PType::member_type e = policy.end(); + for ( typename PType::member_type i = policy.begin() ; i < e ; ++i ) { + functor( i , update , true ); + } + + Kokkos::Impl::FunctorFinal< FunctorType , typename Policy::work_tag >::final( functor , result_ptr ); + } + + // work tag is non-void + template< class PType > + inline + ParallelScan( typename Impl::enable_if< + ( Impl::is_same< PType , Policy >::value && + ! Impl::is_same< typename PType::work_tag , void >::value + ), const FunctorType & >::type functor + , const PType & policy ) + { + pointer_type result_ptr = (pointer_type) + Kokkos::Serial::scratch_memory_resize( ValueTraits::value_size( functor ) , 0 ); + + reference_type update = ValueInit::init( functor , result_ptr ); + + const typename PType::member_type e = policy.end(); + for ( typename PType::member_type i = policy.begin() ; i < e ; ++i ) { + functor( typename PType::work_tag() , i , update , true ); + } + + Kokkos::Impl::FunctorFinal< FunctorType , typename Policy::work_tag >::final( functor , result_ptr ); + } +}; + +} // namespace Impl +} // namespace Kokkos + +/*--------------------------------------------------------------------------*/ +/*--------------------------------------------------------------------------*/ + +namespace Kokkos { +namespace Impl { + +template< class FunctorType , class Arg0 , class Arg1 > +class ParallelFor< FunctorType , Kokkos::TeamPolicy< Arg0 , Arg1 , Kokkos::Serial > > +{ +private: + + typedef Kokkos::TeamPolicy< Arg0 , Arg1 , Kokkos::Serial > Policy ; + + template< class TagType > + KOKKOS_FORCEINLINE_FUNCTION static + void driver( typename Impl::enable_if< Impl::is_same< TagType , void >::value , + const FunctorType & >::type functor + , const typename Policy::member_type & member ) + { functor( member ); } + + template< class TagType > + KOKKOS_FORCEINLINE_FUNCTION static + void driver( typename Impl::enable_if< ! Impl::is_same< TagType , void >::value , + const FunctorType & >::type functor + , const typename Policy::member_type & member ) + { functor( TagType() , member ); } + +public: + + ParallelFor( const FunctorType & functor + , const Policy & policy ) + { + const int shared_size = FunctorTeamShmemSize< FunctorType >::value( functor , policy.team_size() ); + + Kokkos::Serial::scratch_memory_resize( 0 , shared_size ); + + for ( int ileague = 0 ; ileague < policy.league_size() ; ++ileague ) { + ParallelFor::template driver< typename Policy::work_tag > + ( functor , typename Policy::member_type(ileague,policy.league_size(),shared_size) ); + // functor( typename Policy::member_type(ileague,policy.league_size(),shared_size) ); + } + } +}; + +template< class FunctorType , class Arg0 , class Arg1 > +class ParallelReduce< FunctorType , Kokkos::TeamPolicy< Arg0 , Arg1 , Kokkos::Serial > > +{ +private: + + typedef Kokkos::TeamPolicy< Arg0 , Arg1 , Kokkos::Serial > Policy ; + typedef Kokkos::Impl::FunctorValueTraits< FunctorType , typename Policy::work_tag > ValueTraits ; + typedef Kokkos::Impl::FunctorValueInit< FunctorType , typename Policy::work_tag > ValueInit ; + +public: + + typedef typename ValueTraits::pointer_type pointer_type ; + typedef typename ValueTraits::reference_type reference_type ; + +private: + + template< class TagType > + KOKKOS_FORCEINLINE_FUNCTION static + void driver( typename Impl::enable_if< Impl::is_same< TagType , void >::value , + const FunctorType & >::type functor + , const typename Policy::member_type & member + , reference_type update ) + { functor( member , update ); } + + template< class TagType > + KOKKOS_FORCEINLINE_FUNCTION static + void driver( typename Impl::enable_if< ! Impl::is_same< TagType , void >::value , + const FunctorType & >::type functor + , const typename Policy::member_type & member + , reference_type update ) + { functor( TagType() , member , update ); } + +public: + + template< class ViewType > + ParallelReduce( const FunctorType & functor + , const Policy & policy + , const ViewType & result + ) + { + const int reduce_size = ValueTraits::value_size( functor ); + const int shared_size = FunctorTeamShmemSize< FunctorType >::value( functor , policy.team_size() ); + void * const scratch_reduce = Kokkos::Serial::scratch_memory_resize( reduce_size , shared_size ); + + const pointer_type result_ptr = + result.ptr_on_device() ? result.ptr_on_device() + : (pointer_type) scratch_reduce ; + + reference_type update = ValueInit::init( functor , result_ptr ); + + for ( int ileague = 0 ; ileague < policy.league_size() ; ++ileague ) { + ParallelReduce::template driver< typename Policy::work_tag > + ( functor , typename Policy::member_type(ileague,policy.league_size(),shared_size) , update ); + } + + Kokkos::Impl::FunctorFinal< FunctorType , typename Policy::work_tag >::final( functor , result_ptr ); + } +}; + +} // namespace Impl +} // namespace Kokkos + +#ifdef KOKKOS_HAVE_CXX11 + +namespace Kokkos { + +namespace Impl { + template + struct TeamThreadLoopBoundariesStruct { + typedef iType index_type; + enum {start = 0}; + const iType end; + enum {increment = 1}; + const SerialTeamMember& thread; + + KOKKOS_INLINE_FUNCTION + TeamThreadLoopBoundariesStruct (const SerialTeamMember& thread_, const iType& count): + end(count), + thread(thread_) + {} + }; + + template + struct ThreadVectorLoopBoundariesStruct { + typedef iType index_type; + enum {start = 0}; + const iType end; + enum {increment = 1}; + + KOKKOS_INLINE_FUNCTION + ThreadVectorLoopBoundariesStruct (const SerialTeamMember& thread, const iType& count): + end( count ) + {} + }; +} // namespace Impl + +template +KOKKOS_INLINE_FUNCTION +Impl::TeamThreadLoopBoundariesStruct + TeamThreadLoop(const Impl::SerialTeamMember& thread, const iType& count) { + return Impl::TeamThreadLoopBoundariesStruct(thread,count); +} + +template +KOKKOS_INLINE_FUNCTION +Impl::ThreadVectorLoopBoundariesStruct + ThreadVectorLoop(const Impl::SerialTeamMember& thread, const iType& count) { + return Impl::ThreadVectorLoopBoundariesStruct(thread,count); +} + +KOKKOS_INLINE_FUNCTION +Impl::ThreadSingleStruct PerTeam(const Impl::SerialTeamMember& thread) { + return Impl::ThreadSingleStruct(thread); +} + +KOKKOS_INLINE_FUNCTION +Impl::VectorSingleStruct PerThread(const Impl::SerialTeamMember& thread) { + return Impl::VectorSingleStruct(thread); +} + +} // namespace Kokkos + +namespace Kokkos { + + /** \brief Inter-thread parallel_for. Executes lambda(iType i) for each i=0..N-1. + * + * The range i=0..N-1 is mapped to all threads of the the calling thread team. + * This functionality requires C++11 support.*/ +template +KOKKOS_INLINE_FUNCTION +void parallel_for(const Impl::TeamThreadLoopBoundariesStruct& loop_boundaries, const Lambda& lambda) { + for( iType i = loop_boundaries.start; i < loop_boundaries.end; i+=loop_boundaries.increment) + lambda(i); +} + +/** \brief Inter-thread vector parallel_reduce. Executes lambda(iType i, ValueType & val) for each i=0..N-1. + * + * The range i=0..N-1 is mapped to all threads of the the calling thread team and a summation of + * val is performed and put into result. This functionality requires C++11 support.*/ +template< typename iType, class Lambda, typename ValueType > +KOKKOS_INLINE_FUNCTION +void parallel_reduce(const Impl::TeamThreadLoopBoundariesStruct& loop_boundaries, + const Lambda & lambda, ValueType& result) { + + result = ValueType(); + + for( iType i = loop_boundaries.start; i < loop_boundaries.end; i+=loop_boundaries.increment) { + ValueType tmp = ValueType(); + lambda(i,tmp); + result+=tmp; + } + + result = loop_boundaries.thread.team_reduce(result,Impl::JoinAdd()); +} + +/** \brief Intra-thread vector parallel_reduce. Executes lambda(iType i, ValueType & val) for each i=0..N-1. + * + * The range i=0..N-1 is mapped to all vector lanes of the the calling thread and a reduction of + * val is performed using JoinType(ValueType& val, const ValueType& update) and put into init_result. + * The input value of init_result is used as initializer for temporary variables of ValueType. Therefore + * the input value should be the neutral element with respect to the join operation (e.g. '0 for +-' or + * '1 for *'). This functionality requires C++11 support.*/ +template< typename iType, class Lambda, typename ValueType, class JoinType > +KOKKOS_INLINE_FUNCTION +void parallel_reduce(const Impl::TeamThreadLoopBoundariesStruct& loop_boundaries, + const Lambda & lambda, const JoinType& join, ValueType& init_result) { + + ValueType result = init_result; + + for( iType i = loop_boundaries.start; i < loop_boundaries.end; i+=loop_boundaries.increment) { + ValueType tmp = ValueType(); + lambda(i,tmp); + join(result,tmp); + } + + init_result = loop_boundaries.thread.team_reduce(result,Impl::JoinLambdaAdapter(join)); +} + +} //namespace Kokkos + +namespace Kokkos { +/** \brief Intra-thread vector parallel_for. Executes lambda(iType i) for each i=0..N-1. + * + * The range i=0..N-1 is mapped to all vector lanes of the the calling thread. + * This functionality requires C++11 support.*/ +template +KOKKOS_INLINE_FUNCTION +void parallel_for(const Impl::ThreadVectorLoopBoundariesStruct& + loop_boundaries, const Lambda& lambda) { + #ifdef KOKKOS_HAVE_PRAGMA_IVDEP + #pragma ivdep + #endif + for( iType i = loop_boundaries.start; i < loop_boundaries.end; i+=loop_boundaries.increment) + lambda(i); +} + +/** \brief Intra-thread vector parallel_reduce. Executes lambda(iType i, ValueType & val) for each i=0..N-1. + * + * The range i=0..N-1 is mapped to all vector lanes of the the calling thread and a summation of + * val is performed and put into result. This functionality requires C++11 support.*/ +template< typename iType, class Lambda, typename ValueType > +KOKKOS_INLINE_FUNCTION +void parallel_reduce(const Impl::ThreadVectorLoopBoundariesStruct& + loop_boundaries, const Lambda & lambda, ValueType& result) { + result = ValueType(); +#ifdef KOKKOS_HAVE_PRAGMA_IVDEP +#pragma ivdep +#endif + for( iType i = loop_boundaries.start; i < loop_boundaries.end; i+=loop_boundaries.increment) { + ValueType tmp = ValueType(); + lambda(i,tmp); + result+=tmp; + } +} + +/** \brief Intra-thread vector parallel_reduce. Executes lambda(iType i, ValueType & val) for each i=0..N-1. + * + * The range i=0..N-1 is mapped to all vector lanes of the the calling thread and a reduction of + * val is performed using JoinType(ValueType& val, const ValueType& update) and put into init_result. + * The input value of init_result is used as initializer for temporary variables of ValueType. Therefore + * the input value should be the neutral element with respect to the join operation (e.g. '0 for +-' or + * '1 for *'). This functionality requires C++11 support.*/ +template< typename iType, class Lambda, typename ValueType, class JoinType > +KOKKOS_INLINE_FUNCTION +void parallel_reduce(const Impl::ThreadVectorLoopBoundariesStruct& + loop_boundaries, const Lambda & lambda, const JoinType& join, ValueType& init_result) { + + ValueType result = init_result; +#ifdef KOKKOS_HAVE_PRAGMA_IVDEP +#pragma ivdep +#endif + for( iType i = loop_boundaries.start; i < loop_boundaries.end; i+=loop_boundaries.increment) { + ValueType tmp = ValueType(); + lambda(i,tmp); + join(result,tmp); + } + init_result = result; +} + +/** \brief Intra-thread vector parallel exclusive prefix sum. Executes lambda(iType i, ValueType & val, bool final) + * for each i=0..N-1. + * + * The range i=0..N-1 is mapped to all vector lanes in the thread and a scan operation is performed. + * Depending on the target execution space the operator might be called twice: once with final=false + * and once with final=true. When final==true val contains the prefix sum value. The contribution of this + * "i" needs to be added to val no matter whether final==true or not. In a serial execution + * (i.e. team_size==1) the operator is only called once with final==true. Scan_val will be set + * to the final sum value over all vector lanes. + * This functionality requires C++11 support.*/ +template< typename iType, class FunctorType > +KOKKOS_INLINE_FUNCTION +void parallel_scan(const Impl::ThreadVectorLoopBoundariesStruct& + loop_boundaries, const FunctorType & lambda) { + + typedef Kokkos::Impl::FunctorValueTraits< FunctorType , void > ValueTraits ; + typedef typename ValueTraits::value_type value_type ; + + value_type scan_val = value_type(); + +#ifdef KOKKOS_HAVE_PRAGMA_IVDEP +#pragma ivdep +#endif + for( iType i = loop_boundaries.start; i < loop_boundaries.end; i+=loop_boundaries.increment) { + lambda(i,scan_val,true); + } +} + +} // namespace Kokkos + +namespace Kokkos { + +template +KOKKOS_INLINE_FUNCTION +void single(const Impl::VectorSingleStruct& , const FunctorType& lambda) { + lambda(); +} + +template +KOKKOS_INLINE_FUNCTION +void single(const Impl::ThreadSingleStruct& , const FunctorType& lambda) { + lambda(); +} + +template +KOKKOS_INLINE_FUNCTION +void single(const Impl::VectorSingleStruct& , const FunctorType& lambda, ValueType& val) { + lambda(val); +} + +template +KOKKOS_INLINE_FUNCTION +void single(const Impl::ThreadSingleStruct& , const FunctorType& lambda, ValueType& val) { + lambda(val); +} +} +#endif // KOKKOS_HAVE_CXX11 + +#endif // defined( KOKKOS_HAVE_SERIAL ) +#endif /* #define KOKKOS_SERIAL_HPP */ + +//---------------------------------------------------------------------------- +//---------------------------------------------------------------------------- + diff --git a/lib/kokkos/core/src/Kokkos_TaskPolicy.hpp b/lib/kokkos/core/src/Kokkos_TaskPolicy.hpp new file mode 100755 index 0000000000..27139107dc --- /dev/null +++ b/lib/kokkos/core/src/Kokkos_TaskPolicy.hpp @@ -0,0 +1,467 @@ + +/* +//@HEADER +// ************************************************************************ +// +// Kokkos: Manycore Performance-Portable Multidimensional Arrays +// Copyright (2012) Sandia Corporation +// +// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +// the U.S. Government retains certain rights in this software. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// 1. Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// 3. Neither the name of the Corporation nor the names of the +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Questions? Contact H. Carter Edwards (hcedwar@sandia.gov) +// +// ************************************************************************ +//@HEADER +*/ + +// Experimental unified task-data parallel manycore LDRD + +#ifndef KOKKOS_TASKPOLICY_HPP +#define KOKKOS_TASKPOLICY_HPP + +#include +#include +#include +#include + +//---------------------------------------------------------------------------- + +namespace Kokkos { +namespace Impl { + +struct FutureValueTypeIsVoidError {}; + +template < class ExecSpace , class ResultType , class FunctorType > +class TaskMember ; + +template< class ExecPolicy , class ResultType , class FunctorType > +class TaskForEach ; + +template< class ExecPolicy , class ResultType , class FunctorType > +class TaskReduce ; + +template< class ExecPolicy , class ResultType , class FunctorType > +struct TaskScan ; + +} /* namespace Impl */ +} /* namespace Kokkos */ + +//---------------------------------------------------------------------------- + +namespace Kokkos { + +/**\brief States of a task */ +enum TaskState + { TASK_STATE_NULL = 0 ///< Does not exist + , TASK_STATE_CONSTRUCTING = 1 ///< Is under construction + , TASK_STATE_WAITING = 2 ///< Is waiting for execution + , TASK_STATE_EXECUTING = 4 ///< Is executing + , TASK_STATE_COMPLETE = 8 ///< Execution is complete + }; + +template< class Arg1 = void , class Arg2 = void > +class FutureArray ; + +/** + * + * Future< space > // value_type == void + * Future< value > // space == Default + * Future< value , space > + * + */ +template< class Arg1 = void , class Arg2 = void > +class Future { +private: + + template< class , class , class > friend class Impl::TaskMember ; + template< class > friend class TaskPolicy ; + template< class , class > friend class Future ; + template< class , class > friend class FutureArray ; + + // Argument #2, if not void, must be the space. + enum { Arg1_is_space = Impl::is_execution_space< Arg1 >::value }; + enum { Arg2_is_space = Impl::is_execution_space< Arg2 >::value }; + enum { Arg2_is_void = Impl::is_same< Arg2 , void >::value }; + + struct ErrorNoExecutionSpace {}; + + enum { Opt1 = Arg1_is_space && Arg2_is_void + , Opt2 = ! Arg1_is_space && Arg2_is_void + , Opt3 = ! Arg1_is_space && Arg2_is_space + , OptOK = Impl::StaticAssert< Opt1 || Opt2 || Opt3 , ErrorNoExecutionSpace >::value + }; + + typedef typename + Impl::if_c< Opt2 || Opt3 , Arg1 , void >::type + ValueType ; + + typedef typename + Impl::if_c< Opt1 , Arg1 , typename + Impl::if_c< Opt2 , Kokkos::DefaultExecutionSpace , typename + Impl::if_c< Opt3 , Arg2 , void + >::type >::type >::type + ExecutionSpace ; + + typedef Impl::TaskMember< ExecutionSpace , void , void > TaskRoot ; + typedef Impl::TaskMember< ExecutionSpace , ValueType , void > TaskValue ; + + TaskRoot * m_task ; + +public: + + typedef ValueType value_type; + typedef ExecutionSpace execution_space ; + + //---------------------------------------- + + KOKKOS_INLINE_FUNCTION + TaskState get_task_state() const + { return 0 != m_task ? m_task->get_state() : TASK_STATE_NULL ; } + + //---------------------------------------- + + explicit + Future( TaskRoot * task ) + : m_task(0) + { TaskRoot::assign( & m_task , TaskRoot::template verify_type< value_type >( task ) ); } + + //---------------------------------------- + + KOKKOS_INLINE_FUNCTION + ~Future() { TaskRoot::assign( & m_task , 0 , true /* no_throw */ ); } + + //---------------------------------------- + + KOKKOS_INLINE_FUNCTION + Future() : m_task(0) {} + + KOKKOS_INLINE_FUNCTION + Future( const Future & rhs ) + : m_task(0) + { TaskRoot::assign( & m_task , rhs.m_task ); } + + KOKKOS_INLINE_FUNCTION + Future & operator = ( const Future & rhs ) + { TaskRoot::assign( & m_task , rhs.m_task ); return *this ; } + + //---------------------------------------- + + template< class A1 , class A2 > + KOKKOS_INLINE_FUNCTION + Future( const Future & rhs ) + : m_task(0) + { TaskRoot::assign( & m_task , TaskRoot::template verify_type< value_type >( rhs.m_task ) ); } + + template< class A1 , class A2 > + KOKKOS_INLINE_FUNCTION + Future & operator = ( const Future & rhs ) + { TaskRoot::assign( & m_task , TaskRoot::template verify_type< value_type >( rhs.m_task ) ); return *this ; } + + //---------------------------------------- + + typedef typename TaskValue::get_result_type get_result_type ; + + KOKKOS_INLINE_FUNCTION + get_result_type get() const + { return static_cast( m_task )->get(); } +}; + +namespace Impl { + +template< class T > +struct is_future : public Kokkos::Impl::bool_< false > {}; + +template< class Arg0 , class Arg1 > +struct is_future< Kokkos::Future > : public Kokkos::Impl::bool_< true > {}; + +} /* namespace Impl */ +} /* namespace Kokkos */ + +//---------------------------------------------------------------------------- +//---------------------------------------------------------------------------- + +namespace Kokkos { + +template< class Arg1 , class Arg2 > +class FutureArray { +private: + + typedef Future future_type ; + + typedef typename future_type::execution_space ExecutionSpace ; + typedef typename ExecutionSpace::memory_space MemorySpace ; + + typedef Impl::TaskMember< ExecutionSpace , void , void > TaskRoot ; + + future_type * m_future ; + + //---------------------------------------- + +public: + + typedef ExecutionSpace execution_space ; + typedef future_type value_type ; + + //---------------------------------------- + + KOKKOS_INLINE_FUNCTION + size_t size() const + { return m_future ? reinterpret_cast(m_future->m_task) : size_t(0) ; } + + KOKKOS_INLINE_FUNCTION + value_type & operator[]( const int i ) const + { return m_future[i+1]; } + + //---------------------------------------- + + KOKKOS_INLINE_FUNCTION + ~FutureArray() + { +#if defined( KOKKOS_ACTIVE_EXECUTION_MEMORY_SPACE_HOST ) + if ( m_future ) { + const size_t n = size(); + for ( size_t i = 1 ; i <= n ; ++i ) { + TaskRoot::assign( & m_future[i].m_task , 0 ); + } + m_future[0].m_task = 0 ; + MemorySpace::decrement( m_future ); + } +#endif + } + + KOKKOS_INLINE_FUNCTION + FutureArray() : m_future(0) {} + + inline + FutureArray( const size_t n ) + : m_future(0) + { + if ( n ) { + m_future = (future_type *) MemorySpace::allocate( "FutureArray" , sizeof(future_type) * ( n + 1 ) ); + for ( size_t i = 0 ; i <= n ; ++i ) m_future[i].m_task = 0 ; + } + } + + KOKKOS_INLINE_FUNCTION + FutureArray( const FutureArray & rhs ) + : m_future( rhs.m_future ) + { +#if defined( KOKKOS_ACTIVE_EXECUTION_MEMORY_SPACE_HOST ) + MemorySpace::increment( m_future ); +#endif + } + + KOKKOS_INLINE_FUNCTION + FutureArray & operator = ( const FutureArray & rhs ) + { +#if defined( KOKKOS_ACTIVE_EXECUTION_MEMORY_SPACE_HOST ) + MemorySpace::decrement( m_future ); + MemorySpace::increment( rhs.m_future ); +#endif + m_future = rhs.m_future ; + return *this ; + } +}; + +} /* namespace Kokkos */ + +//---------------------------------------------------------------------------- + +namespace Kokkos { + +/** \brief If the argument is an execution space then a serial task in that space */ +template< class Arg0 = Kokkos::DefaultExecutionSpace > +class TaskPolicy { +public: + + typedef typename Arg0::execution_space execution_space ; + + //---------------------------------------- + /** \brief Create a serial task with storage for dependences. + * + * Postcondition: Task is in the 'constructing' state. + */ + template< class FunctorType > + Future< typename FunctorType::value_type , execution_space > + create( const FunctorType & functor + , const unsigned dependence_capacity /* = default */ ) const ; + + /** \brief Create a foreach task with storage for dependences. */ + template< class ExecPolicy , class FunctorType > + Future< typename FunctorType::value_type , execution_space > + create_foreach( const ExecPolicy & policy + , const FunctorType & functor + , const unsigned dependence_capacity /* = default */ ) const ; + + /** \brief Create a reduce task with storage for dependences. */ + template< class ExecPolicy , class FunctorType > + Future< typename FunctorType::value_type , execution_space > + create_reduce( const ExecPolicy & policy + , const FunctorType & functor + , const unsigned dependence_capacity /* = default */ ) const ; + + /** \brief Create a scan task with storage for dependences. */ + template< class ExecPolicy , class FunctorType > + Future< typename FunctorType::value_type , execution_space > + create_scan( const ExecPolicy & policy + , const FunctorType & functor + , const unsigned dependence_capacity /* = default */ ) const ; + + /** \brief Set dependence that 'after' cannot start execution + * until 'before' has completed. + * + * Precondition: The 'after' task must be in then 'Constructing' state. + */ + template< class TA , class TB > + void set_dependence( const Future & after + , const Future & before ) const ; + + /** \brief Spawn a task in the 'Constructing' state + * + * Precondition: Task is in the 'constructing' state. + * Postcondition: Task is waiting, executing, or complete. + */ + template< class T > + const Future & + spawn( const Future & ) const ; + + //---------------------------------------- + /** \brief Query dependence of an executing task */ + + template< class FunctorType > + Future< execution_space > + get_dependence( FunctorType * , const int ) const ; + + //---------------------------------------- + /** \brief Clear current dependences of an executing task + * in preparation for setting new dependences and + * respawning. + * + * Precondition: The functor must be a task in the executing state. + */ + template< class FunctorType > + void clear_dependence( FunctorType * ) const ; + + /** \brief Set dependence that 'after' cannot start execution + * until 'before' has completed. + * + * The 'after' functor must be in the executing state + */ + template< class FunctorType , class TB > + void set_dependence( FunctorType * after + , const Future & before ) const ; + + /** \brief Respawn (reschedule) an executing task to be called again + * after all dependences have completed. + */ + template< class FunctorType > + void respawn( FunctorType * ) const ; +}; + +//---------------------------------------------------------------------------- +/** \brief Create and spawn a single-thread task */ +template< class ExecSpace , class FunctorType > +inline +Future< typename FunctorType::value_type , ExecSpace > +spawn( TaskPolicy & policy , const FunctorType & functor ) +{ return policy.spawn( policy.create( functor ) ); } + +/** \brief Create and spawn a single-thread task with dependences */ +template< class ExecSpace , class FunctorType , class Arg0 , class Arg1 > +inline +Future< typename FunctorType::value_type , ExecSpace > +spawn( TaskPolicy & policy + , const FunctorType & functor + , const Future & before_0 + , const Future & before_1 ) +{ + Future< typename FunctorType::value_type , ExecSpace > f ; + f = policy.create( functor , 2 ); + policy.add_dependence( f , before_0 ); + policy.add_dependence( f , before_1 ); + policy.spawn( f ); + return f ; +} + +//---------------------------------------------------------------------------- +/** \brief Create and spawn a parallel_for task */ +template< class ExecSpace , class ParallelPolicyType , class FunctorType > +inline +Future< typename FunctorType::value_type , ExecSpace > +spawn_foreach( TaskPolicy & task_policy + , const ParallelPolicyType & parallel_policy + , const FunctorType & functor ) +{ return task_policy.spawn( task_policy.create_foreach( parallel_policy , functor ) ); } + +/** \brief Create and spawn a parallel_reduce task */ +template< class ExecSpace , class ParallelPolicyType , class FunctorType > +inline +Future< typename FunctorType::value_type , ExecSpace > +spawn_reduce( TaskPolicy & task_policy + , const ParallelPolicyType & parallel_policy + , const FunctorType & functor ) +{ return task_policy.spawn( task_policy.create_reduce( parallel_policy , functor ) ); } + +//---------------------------------------------------------------------------- +/** \brief Respawn a task functor with dependences */ +template< class ExecSpace , class FunctorType , class Arg0 , class Arg1 > +inline +void respawn( TaskPolicy & policy + , FunctorType * functor + , const Future & before_0 + , const Future & before_1 + ) +{ + policy.clear_dependence( functor ); + policy.add_dependence( functor , before_0 ); + policy.add_dependence( functor , before_1 ); + policy.respawn( functor ); +} + +//---------------------------------------------------------------------------- + +template< class ExecSpace > +void wait( TaskPolicy< ExecSpace > & ); + +template< class A0 , class A1 > +inline +void wait( const Future & future ) +{ + wait( Future< void , typename Future::execution_space >( future ) ); +} + +} /* namespace Kokkos */ + +//---------------------------------------------------------------------------- +//---------------------------------------------------------------------------- + +#endif /* #define KOKKOS_TASKPOLICY_HPP */ + diff --git a/lib/kokkos/core/src/Kokkos_Threads.hpp b/lib/kokkos/core/src/Kokkos_Threads.hpp new file mode 100755 index 0000000000..3d6a64c4b7 --- /dev/null +++ b/lib/kokkos/core/src/Kokkos_Threads.hpp @@ -0,0 +1,214 @@ +/* +//@HEADER +// ************************************************************************ +// +// Kokkos: Manycore Performance-Portable Multidimensional Arrays +// Copyright (2012) Sandia Corporation +// +// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +// the U.S. Government retains certain rights in this software. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// 1. Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// 3. Neither the name of the Corporation nor the names of the +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Questions? Contact H. Carter Edwards (hcedwar@sandia.gov) +// +// ************************************************************************ +//@HEADER +*/ + +#ifndef KOKKOS_THREADS_HPP +#define KOKKOS_THREADS_HPP + +#include + +#if defined( KOKKOS_HAVE_PTHREAD ) + +#include +#include +#include +#include +#include +#include +#include + +/*--------------------------------------------------------------------------*/ + +namespace Kokkos { +namespace Impl { +class ThreadsExec ; +} // namespace Impl +} // namespace Kokkos + +/*--------------------------------------------------------------------------*/ + +namespace Kokkos { + +/** \brief Execution space for a pool of Pthreads or C11 threads on a CPU. */ +class Threads { +public: + //! \name Type declarations that all Kokkos devices must provide. + //@{ + //! Tag this class as a kokkos execution space + typedef Threads execution_space ; + typedef Kokkos::HostSpace memory_space ; + typedef Kokkos::LayoutRight array_layout ; + typedef memory_space::size_type size_type ; + + typedef ScratchMemorySpace< Threads > scratch_memory_space ; + + //! For backward compatibility + typedef Threads device_type ; + + //@} + /*------------------------------------------------------------------------*/ + //! \name Static functions that all Kokkos devices must implement. + //@{ + + /// \brief True if and only if this method is being called in a + /// thread-parallel function. + static int in_parallel(); + + /** \brief Set the device in a "sleep" state. + * + * This function sets the device in a "sleep" state in which it is + * not ready for work. This may consume less resources than if the + * device were in an "awake" state, but it may also take time to + * bring the device from a sleep state to be ready for work. + * + * \return True if the device is in the "sleep" state, else false if + * the device is actively working and could not enter the "sleep" + * state. + */ + static bool sleep(); + + /// \brief Wake the device from the 'sleep' state so it is ready for work. + /// + /// \return True if the device is in the "ready" state, else "false" + /// if the device is actively working (which also means that it's + /// awake). + static bool wake(); + + /// \brief Wait until all dispatched functors complete. + /// + /// The parallel_for or parallel_reduce dispatch of a functor may + /// return asynchronously, before the functor completes. This + /// method does not return until all dispatched functors on this + /// device have completed. + static void fence(); + + /// \brief Free any resources being consumed by the device. + /// + /// For the Threads device, this terminates spawned worker threads. + static void finalize(); + + /// \brief Print configuration information to the given output stream. + static void print_configuration( std::ostream & , const bool detail = false ); + + //@} + /*------------------------------------------------------------------------*/ + /*------------------------------------------------------------------------*/ + //! \name Space-specific functions + //@{ + + /** \brief Initialize the device in the "ready to work" state. + * + * The device is initialized in a "ready to work" or "awake" state. + * This state reduces latency and thus improves performance when + * dispatching work. However, the "awake" state consumes resources + * even when no work is being done. You may call sleep() to put + * the device in a "sleeping" state that does not consume as many + * resources, but it will take time (latency) to awaken the device + * again (via the wake()) method so that it is ready for work. + * + * Teams of threads are distributed as evenly as possible across + * the requested number of numa regions and cores per numa region. + * A team will not be split across a numa region. + * + * If the 'use_' arguments are not supplied the hwloc is queried + * to use all available cores. + */ + static void initialize( unsigned threads_count = 1 , + unsigned use_numa_count = 0 , + unsigned use_cores_per_numa = 0 , + bool allow_asynchronous_threadpool = false ); + + static int is_initialized(); + + static Threads & instance( int = 0 ); + + //---------------------------------------- + + static int thread_pool_size( int depth = 0 ); +#if defined( KOKKOS_ACTIVE_EXECUTION_MEMORY_SPACE_HOST ) + static int thread_pool_rank(); +#else + KOKKOS_INLINE_FUNCTION static int thread_pool_rank() { return 0 ; } +#endif + + inline static unsigned max_hardware_threads() { return thread_pool_size(0); } + KOKKOS_INLINE_FUNCTION static unsigned hardware_thread_id() { return thread_pool_rank(); } + + //@} + //---------------------------------------- +}; + +} // namespace Kokkos + +/*--------------------------------------------------------------------------*/ + +namespace Kokkos { +namespace Impl { + +template<> +struct VerifyExecutionCanAccessMemorySpace + < Kokkos::Threads::memory_space + , Kokkos::Threads::scratch_memory_space + > +{ + enum { value = true }; + inline static void verify( void ) { } + inline static void verify( const void * ) { } +}; + +} // namespace Impl +} // namespace Kokkos + +/*--------------------------------------------------------------------------*/ + +#include +#include +#include +#include + +//---------------------------------------------------------------------------- +//---------------------------------------------------------------------------- + +#endif /* #if defined( KOKKOS_HAVE_PTHREAD ) */ +#endif /* #define KOKKOS_THREADS_HPP */ + + diff --git a/lib/kokkos/core/src/Kokkos_Vectorization.hpp b/lib/kokkos/core/src/Kokkos_Vectorization.hpp new file mode 100755 index 0000000000..8a91f25298 --- /dev/null +++ b/lib/kokkos/core/src/Kokkos_Vectorization.hpp @@ -0,0 +1,100 @@ +/* +//@HEADER +// ************************************************************************ +// +// Kokkos: Manycore Performance-Portable Multidimensional Arrays +// Copyright (2012) Sandia Corporation +// +// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +// the U.S. Government retains certain rights in this software. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// 1. Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// 3. Neither the name of the Corporation nor the names of the +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Questions? Contact H. Carter Edwards (hcedwar@sandia.gov) +// +// ************************************************************************ +//@HEADER +*/ + +/// \file Kokkos_Vectorization.hpp +/// \brief Declaration and definition of Kokkos::Vectorization interface. +#ifndef KOKKOS_VECTORIZATION_HPP +#define KOKKOS_VECTORIZATION_HPP + +#include +#include + +namespace Kokkos { +template +struct Vectorization { + typedef Kokkos::TeamPolicy< Space > team_policy ; + typedef typename team_policy::member_type team_member ; + + enum {increment = 1}; + + KOKKOS_FORCEINLINE_FUNCTION + static int begin() { return 0;} + + KOKKOS_FORCEINLINE_FUNCTION + static int thread_rank(const team_member &dev) { + return dev.team_rank(); + } + + KOKKOS_FORCEINLINE_FUNCTION + static int team_rank(const team_member &dev) { + return dev.team_rank()/increment; + } + + KOKKOS_FORCEINLINE_FUNCTION + static int team_size(const team_member &dev) { + return dev.team_size()/increment; + } + + KOKKOS_FORCEINLINE_FUNCTION + static int global_thread_rank(const team_member &dev) { + return (dev.league_rank()*dev.team_size()+dev.team_rank()); + } + + KOKKOS_FORCEINLINE_FUNCTION + static bool is_lane_0(const team_member &dev) { + return true; + } + + template + KOKKOS_FORCEINLINE_FUNCTION + static Scalar reduce(const Scalar& val) { + return val; + } +}; +} + +#if defined( KOKKOS_HAVE_CUDA ) +#include +#endif + +#endif diff --git a/lib/kokkos/core/src/Kokkos_View.hpp b/lib/kokkos/core/src/Kokkos_View.hpp new file mode 100755 index 0000000000..c215214bff --- /dev/null +++ b/lib/kokkos/core/src/Kokkos_View.hpp @@ -0,0 +1,1863 @@ +/* +//@HEADER +// ************************************************************************ +// +// Kokkos: Manycore Performance-Portable Multidimensional Arrays +// Copyright (2012) Sandia Corporation +// +// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +// the U.S. Government retains certain rights in this software. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// 1. Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// 3. Neither the name of the Corporation nor the names of the +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Questions? Contact H. Carter Edwards (hcedwar@sandia.gov) +// +// ************************************************************************ +//@HEADER +*/ + +#ifndef KOKKOS_VIEW_HPP +#define KOKKOS_VIEW_HPP + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +//---------------------------------------------------------------------------- +//---------------------------------------------------------------------------- + +namespace Kokkos { +namespace Impl { + +/** \brief View specialization mapping of view traits to a specialization tag */ +template< class ValueType , + class ArraySpecialize , + class ArrayLayout , + class MemorySpace , + class MemoryTraits > +struct ViewSpecialize ; + +/** \brief Defines the type of a subview given a source view type + * and subview argument types. + */ +template< class SrcViewType + , class Arg0Type + , class Arg1Type + , class Arg2Type + , class Arg3Type + , class Arg4Type + , class Arg5Type + , class Arg6Type + , class Arg7Type + > +struct ViewSubview /* { typedef ... type ; } */ ; + +template< class DstViewSpecialize , + class SrcViewSpecialize = void , + class Enable = void > +struct ViewAssignment ; + +template< class DstMemorySpace , class SrcMemorySpace > +struct DeepCopy ; + +} /* namespace Impl */ +} // namespace Kokkos + +//---------------------------------------------------------------------------- +//---------------------------------------------------------------------------- + +namespace Kokkos { + +/** \class ViewTraits + * \brief Traits class for accessing attributes of a View. + * + * This is an implementation detail of View. It is only of interest + * to developers implementing a new specialization of View. + * + * Template argument permutations: + * - View< DataType , void , void , void > + * - View< DataType , Space , void , void > + * - View< DataType , Space , MemoryTraits , void > + * - View< DataType , Space , void , MemoryTraits > + * - View< DataType , ArrayLayout , void , void > + * - View< DataType , ArrayLayout , Space , void > + * - View< DataType , ArrayLayout , MemoryTraits , void > + * - View< DataType , ArrayLayout , Space , MemoryTraits > + * - View< DataType , MemoryTraits , void , void > + */ + +template< class DataType , + class Arg1 = void , + class Arg2 = void , + class Arg3 = void > +class ViewTraits { +private: + + // Layout, Space, and MemoryTraits are optional + // but need to appear in that order. That means Layout + // can only be Arg1, Space can be Arg1 or Arg2, and + // MemoryTraits can be Arg1, Arg2 or Arg3 + + enum { Arg1IsLayout = Impl::is_array_layout::value }; + + enum { Arg1IsSpace = Impl::is_space::value }; + enum { Arg2IsSpace = Impl::is_space::value }; + + enum { Arg1IsMemoryTraits = Impl::is_memory_traits::value }; + enum { Arg2IsMemoryTraits = Impl::is_memory_traits::value }; + enum { Arg3IsMemoryTraits = Impl::is_memory_traits::value }; + + enum { Arg1IsVoid = Impl::is_same< Arg1 , void >::value }; + enum { Arg2IsVoid = Impl::is_same< Arg2 , void >::value }; + enum { Arg3IsVoid = Impl::is_same< Arg3 , void >::value }; + + // Arg1 is Layout, Space, MemoryTraits, or void + typedef typename + Impl::StaticAssert< + ( 1 == Arg1IsLayout + Arg1IsSpace + Arg1IsMemoryTraits + Arg1IsVoid ) + , Arg1 >::type Arg1Verified ; + + // If Arg1 is Layout then Arg2 is Space, MemoryTraits, or void + // If Arg1 is Space then Arg2 is MemoryTraits or void + // If Arg1 is MemoryTraits then Arg2 is void + // If Arg1 is Void then Arg2 is void + typedef typename + Impl::StaticAssert< + ( Arg1IsLayout && ( 1 == Arg2IsSpace + Arg2IsMemoryTraits + Arg2IsVoid ) ) || + ( Arg1IsSpace && ( 0 == Arg2IsSpace ) && ( 1 == Arg2IsMemoryTraits + Arg2IsVoid ) ) || + ( Arg1IsMemoryTraits && Arg2IsVoid ) || + ( Arg1IsVoid && Arg2IsVoid ) + , Arg2 >::type Arg2Verified ; + + // Arg3 is MemoryTraits or void and at most one argument is MemoryTraits + typedef typename + Impl::StaticAssert< + ( 1 == Arg3IsMemoryTraits + Arg3IsVoid ) && + ( Arg1IsMemoryTraits + Arg2IsMemoryTraits + Arg3IsMemoryTraits <= 1 ) + , Arg3 >::type Arg3Verified ; + + // Arg1 or Arg2 may have execution and memory spaces + typedef typename Impl::if_c<( Arg1IsSpace ), Arg1Verified , + typename Impl::if_c<( Arg2IsSpace ), Arg2Verified , + Kokkos::DefaultExecutionSpace + >::type >::type::execution_space ExecutionSpace ; + + typedef typename Impl::if_c<( Arg1IsSpace ), Arg1Verified , + typename Impl::if_c<( Arg2IsSpace ), Arg2Verified , + Kokkos::DefaultExecutionSpace + >::type >::type::memory_space MemorySpace ; + + typedef typename Impl::is_space< + typename Impl::if_c<( Arg1IsSpace ), Arg1Verified , + typename Impl::if_c<( Arg2IsSpace ), Arg2Verified , + Kokkos::DefaultExecutionSpace + >::type >::type >::host_mirror_space HostMirrorSpace ; + + // Arg1 may be array layout + typedef typename Impl::if_c< Arg1IsLayout , Arg1Verified , + typename ExecutionSpace::array_layout + >::type ArrayLayout ; + + // Arg1, Arg2, or Arg3 may be memory traits + typedef typename Impl::if_c< Arg1IsMemoryTraits , Arg1Verified , + typename Impl::if_c< Arg2IsMemoryTraits , Arg2Verified , + typename Impl::if_c< Arg3IsMemoryTraits , Arg3Verified , + MemoryManaged + >::type >::type >::type MemoryTraits ; + + typedef Impl::AnalyzeShape analysis ; + +public: + + //------------------------------------ + // Data type traits: + + typedef DataType data_type ; + typedef typename analysis::const_type const_data_type ; + typedef typename analysis::non_const_type non_const_data_type ; + + //------------------------------------ + // Array of intrinsic scalar type traits: + + typedef typename analysis::array_intrinsic_type array_intrinsic_type ; + typedef typename analysis::const_array_intrinsic_type const_array_intrinsic_type ; + typedef typename analysis::non_const_array_intrinsic_type non_const_array_intrinsic_type ; + + //------------------------------------ + // Value type traits: + + typedef typename analysis::value_type value_type ; + typedef typename analysis::const_value_type const_value_type ; + typedef typename analysis::non_const_value_type non_const_value_type ; + + //------------------------------------ + // Layout and shape traits: + + typedef ArrayLayout array_layout ; + typedef typename analysis::shape shape_type ; + + enum { rank = shape_type::rank }; + enum { rank_dynamic = shape_type::rank_dynamic }; + + //------------------------------------ + // Execution space, memory space, memory access traits, and host mirror space. + + typedef ExecutionSpace execution_space ; + typedef MemorySpace memory_space ; + typedef MemoryTraits memory_traits ; + typedef HostMirrorSpace host_mirror_space ; + + typedef typename memory_space::size_type size_type ; + + enum { is_hostspace = Impl::is_same< memory_space , HostSpace >::value }; + enum { is_managed = memory_traits::Unmanaged == 0 }; + enum { is_random_access = memory_traits::RandomAccess == 1 }; + + //------------------------------------ + + typedef ExecutionSpace device_type ; // for backward compatibility, to be removed + + //------------------------------------ + // Specialization tag: + + typedef typename + Impl::ViewSpecialize< value_type + , typename analysis::specialize + , array_layout + , memory_space + , memory_traits + >::type specialize ; +}; + +} /* namespace Kokkos */ + +//---------------------------------------------------------------------------- +//---------------------------------------------------------------------------- + +namespace Kokkos { +namespace Impl { + +class ViewDefault {}; + +/** \brief Default view specialization has LayoutLeft, LayoutRight, or LayoutStride. + */ +template< class ValueType , class MemorySpace , class MemoryTraits > +struct ViewSpecialize< ValueType , void , LayoutLeft , MemorySpace , MemoryTraits > +{ typedef ViewDefault type ; }; + +template< class ValueType , class MemorySpace , class MemoryTraits > +struct ViewSpecialize< ValueType , void , LayoutRight , MemorySpace , MemoryTraits > +{ typedef ViewDefault type ; }; + +template< class ValueType , class MemorySpace , class MemoryTraits > +struct ViewSpecialize< ValueType , void , LayoutStride , MemorySpace , MemoryTraits > +{ typedef ViewDefault type ; }; + +} /* namespace Impl */ +} /* namespace Kokkos */ + +//---------------------------------------------------------------------------- +//---------------------------------------------------------------------------- + +namespace Kokkos { +namespace Impl { + +/** \brief Types for compile-time detection of View usage errors */ +namespace ViewError { + +struct allocation_constructor_requires_managed {}; +struct allocation_constructor_requires_nonconst {}; +struct user_pointer_constructor_requires_unmanaged {}; +struct device_shmem_constructor_requires_unmanaged {}; + +struct scalar_operator_called_from_non_scalar_view {}; + +} /* namespace ViewError */ + +//---------------------------------------------------------------------------- +/** \brief Enable view parentheses operator for + * match of layout and integral arguments. + * If correct rank define type from traits, + * otherwise define type as an error message. + */ +template< class ReturnType , class Traits , class Layout , unsigned Rank , + typename iType0 = int , typename iType1 = int , + typename iType2 = int , typename iType3 = int , + typename iType4 = int , typename iType5 = int , + typename iType6 = int , typename iType7 = int , + class Enable = void > +struct ViewEnableArrayOper ; + +template< class ReturnType , class Traits , class Layout , unsigned Rank , + typename iType0 , typename iType1 , + typename iType2 , typename iType3 , + typename iType4 , typename iType5 , + typename iType6 , typename iType7 > +struct ViewEnableArrayOper< + ReturnType , Traits , Layout , Rank , + iType0 , iType1 , iType2 , iType3 , + iType4 , iType5 , iType6 , iType7 , + typename enable_if< + iType0(0) == 0 && iType1(0) == 0 && iType2(0) == 0 && iType3(0) == 0 && + iType4(0) == 0 && iType5(0) == 0 && iType6(0) == 0 && iType7(0) == 0 && + is_same< typename Traits::array_layout , Layout >::value && + ( unsigned(Traits::rank) == Rank ) + >::type > +{ + typedef ReturnType type ; +}; + +} /* namespace Impl */ +} /* namespace Kokkos */ + +//---------------------------------------------------------------------------- +//---------------------------------------------------------------------------- + +namespace Kokkos { + +/** \class View + * \brief View to an array of data. + * + * A View represents an array of one or more dimensions. + * For details, please refer to Kokkos' tutorial materials. + * + * \section Kokkos_View_TemplateParameters Template parameters + * + * This class has both required and optional template parameters. The + * \c DataType parameter must always be provided, and must always be + * first. The parameters \c Arg1Type, \c Arg2Type, and \c Arg3Type are + * placeholders for different template parameters. The default value + * of the fifth template parameter \c Specialize suffices for most use + * cases. When explaining the template parameters, we won't refer to + * \c Arg1Type, \c Arg2Type, and \c Arg3Type; instead, we will refer + * to the valid categories of template parameters, in whatever order + * they may occur. + * + * Valid ways in which template arguments may be specified: + * - View< DataType , Space > + * - View< DataType , Space , MemoryTraits > + * - View< DataType , Space , void , MemoryTraits > + * - View< DataType , Layout , Space > + * - View< DataType , Layout , Space , MemoryTraits > + * + * \tparam DataType (required) This indicates both the type of each + * entry of the array, and the combination of compile-time and + * run-time array dimension(s). For example, double* + * indicates a one-dimensional array of \c double with run-time + * dimension, and int*[3] a two-dimensional array of \c int + * with run-time first dimension and compile-time second dimension + * (of 3). In general, the run-time dimensions (if any) must go + * first, followed by zero or more compile-time dimensions. For + * more examples, please refer to the tutorial materials. + * + * \tparam Space (required) The memory space. + * + * \tparam Layout (optional) The array's layout in memory. For + * example, LayoutLeft indicates a column-major (Fortran style) + * layout, and LayoutRight a row-major (C style) layout. If not + * specified, this defaults to the preferred layout for the + * Space. + * + * \tparam MemoryTraits (optional) Assertion of the user's intended + * access behavior. For example, RandomAccess indicates read-only + * access with limited spatial locality, and Unmanaged lets users + * wrap externally allocated memory in a View without automatic + * deallocation. + * + * \section Kokkos_View_MT MemoryTraits discussion + * + * \subsection Kokkos_View_MT_Interp MemoryTraits interpretation depends on Space + * + * Some \c MemoryTraits options may have different interpretations for + * different \c Space types. For example, with the Cuda device, + * \c RandomAccess tells Kokkos to fetch the data through the texture + * cache, whereas the non-GPU devices have no such hardware construct. + * + * \subsection Kokkos_View_MT_PrefUse Preferred use of MemoryTraits + * + * Users should defer applying the optional \c MemoryTraits parameter + * until the point at which they actually plan to rely on it in a + * computational kernel. This minimizes the number of template + * parameters exposed in their code, which reduces the cost of + * compilation. Users may always assign a View without specified + * \c MemoryTraits to a compatible View with that specification. + * For example: + * \code + * // Pass in the simplest types of View possible. + * void + * doSomething (View out, + * View in) + * { + * // Assign the "generic" View in to a RandomAccess View in_rr. + * // Note that RandomAccess View objects must have const data. + * View in_rr = in; + * // ... do something with in_rr and out ... + * } + * \endcode + */ +template< class DataType , + class Arg1Type = void , /* ArrayLayout, SpaceType, or MemoryTraits */ + class Arg2Type = void , /* SpaceType or MemoryTraits */ + class Arg3Type = void , /* MemoryTraits */ + class Specialize = + typename ViewTraits::specialize > +class View ; + +namespace Impl { + +template< class C > +struct is_view : public bool_< false > {}; + +template< class D , class A1 , class A2 , class A3 , class S > +struct is_view< View< D , A1 , A2 , A3 , S > > : public bool_< true > {}; + +} + +//---------------------------------------------------------------------------- + +template< class DataType , + class Arg1Type , + class Arg2Type , + class Arg3Type > +class View< DataType , Arg1Type , Arg2Type , Arg3Type , Impl::ViewDefault > + : public ViewTraits< DataType , Arg1Type , Arg2Type, Arg3Type > +{ +public: + + typedef ViewTraits< DataType , Arg1Type , Arg2Type, Arg3Type > traits ; + +private: + + // Assignment of compatible views requirement: + template< class , class , class , class , class > friend class View ; + + // Assignment of compatible subview requirement: + template< class , class , class > friend struct Impl::ViewAssignment ; + + // Dimensions, cardinality, capacity, and offset computation for + // multidimensional array view of contiguous memory. + // Inherits from Impl::Shape + typedef Impl::ViewOffset< typename traits::shape_type + , typename traits::array_layout + > offset_map_type ; + + // Intermediary class for data management and access + typedef Impl::ViewDataManagement< traits > view_data_management ; + + //---------------------------------------- + // Data members: + + typename view_data_management::handle_type m_ptr_on_device ; + offset_map_type m_offset_map ; + view_data_management m_management ; + + //---------------------------------------- + +public: + + /** return type for all indexing operators */ + typedef typename view_data_management::return_type reference_type ; + + typedef View< typename traits::array_intrinsic_type , + typename traits::array_layout , + typename traits::execution_space , + typename traits::memory_traits > array_type ; + + typedef View< typename traits::const_data_type , + typename traits::array_layout , + typename traits::execution_space , + typename traits::memory_traits > const_type ; + + typedef View< typename traits::non_const_data_type , + typename traits::array_layout , + typename traits::execution_space , + typename traits::memory_traits > non_const_type ; + + typedef View< typename traits::non_const_data_type , + typename traits::array_layout , + typename traits::host_mirror_space , + void > HostMirror ; + + //------------------------------------ + // Shape + + enum { Rank = traits::rank }; + + KOKKOS_INLINE_FUNCTION offset_map_type shape() const { return m_offset_map ; } + KOKKOS_INLINE_FUNCTION typename traits::size_type dimension_0() const { return m_offset_map.N0 ; } + KOKKOS_INLINE_FUNCTION typename traits::size_type dimension_1() const { return m_offset_map.N1 ; } + KOKKOS_INLINE_FUNCTION typename traits::size_type dimension_2() const { return m_offset_map.N2 ; } + KOKKOS_INLINE_FUNCTION typename traits::size_type dimension_3() const { return m_offset_map.N3 ; } + KOKKOS_INLINE_FUNCTION typename traits::size_type dimension_4() const { return m_offset_map.N4 ; } + KOKKOS_INLINE_FUNCTION typename traits::size_type dimension_5() const { return m_offset_map.N5 ; } + KOKKOS_INLINE_FUNCTION typename traits::size_type dimension_6() const { return m_offset_map.N6 ; } + KOKKOS_INLINE_FUNCTION typename traits::size_type dimension_7() const { return m_offset_map.N7 ; } + KOKKOS_INLINE_FUNCTION typename traits::size_type size() const { return m_offset_map.cardinality(); } + + template< typename iType > + KOKKOS_INLINE_FUNCTION + typename traits::size_type dimension( const iType & i ) const + { return Impl::dimension( m_offset_map , i ); } + + //------------------------------------ + // Destructor, constructors, assignment operators: + + KOKKOS_INLINE_FUNCTION + ~View() + { m_management.decrement( m_ptr_on_device ); } + + KOKKOS_INLINE_FUNCTION + View() + : m_ptr_on_device((typename traits::value_type*) NULL) + , m_offset_map() + , m_management() + { m_offset_map.assign(0, 0,0,0,0,0,0,0,0); } + + KOKKOS_INLINE_FUNCTION + View( const View & rhs ) + : m_ptr_on_device((typename traits::value_type*) NULL) + , m_offset_map() + , m_management() + { + (void) Impl::ViewAssignment< + typename traits::specialize , + typename traits::specialize >( *this , rhs ); + } + + KOKKOS_INLINE_FUNCTION + View & operator = ( const View & rhs ) + { + (void) Impl::ViewAssignment< + typename traits::specialize , + typename traits::specialize >( *this , rhs ); + return *this ; + } + + //------------------------------------ + // Construct or assign compatible view: + + template< class RT , class RL , class RD , class RM , class RS > + KOKKOS_INLINE_FUNCTION + View( const View & rhs ) + : m_ptr_on_device((typename traits::value_type*) NULL) + , m_offset_map() + , m_management() + { + (void) Impl::ViewAssignment< + typename traits::specialize , RS >( *this , rhs ); + } + + template< class RT , class RL , class RD , class RM , class RS > + KOKKOS_INLINE_FUNCTION + View & operator = ( const View & rhs ) + { + (void) Impl::ViewAssignment< + typename traits::specialize , RS >( *this , rhs ); + return *this ; + } + + //------------------------------------ + /**\brief Allocation of a managed view with possible alignment padding. + * + * Allocation properties for allocating and initializing to the default value_type: + * Kokkos::ViewAllocate() + * Kokkos::ViewAllocate("label") OR "label" + * Kokkos::ViewAllocate(std::string("label")) OR std::string("label") + * + * Allocation properties for allocating and bypassing initialization: + * Kokkos::ViewAllocateWithoutInitializing() + * Kokkos::ViewAllocateWithoutInitializing("label") + */ + + template< class AllocationProperties > + explicit inline + View( const AllocationProperties & prop , + // Impl::ViewAllocProp::size_type exists when the traits and allocation properties + // are valid for allocating viewed memory. + const typename Impl::ViewAllocProp< traits , AllocationProperties >::size_type n0 = 0 , + const size_t n1 = 0 , + const size_t n2 = 0 , + const size_t n3 = 0 , + const size_t n4 = 0 , + const size_t n5 = 0 , + const size_t n6 = 0 , + const size_t n7 = 0 , + const size_t n8 = 0 ) + : m_ptr_on_device(0) + , m_offset_map() + , m_management() + { + typedef Impl::ViewAllocProp< traits , AllocationProperties > Alloc ; + + m_offset_map.assign( n0, n1, n2, n3, n4, n5, n6, n7, n8 ); + if(Alloc::AllowPadding) + m_offset_map.set_padding(); + + m_ptr_on_device = view_data_management::template allocate< Alloc::Initialize >( Alloc::label(prop) , m_offset_map ); + } + + template< class AllocationProperties > + explicit inline + View( const AllocationProperties & prop , + const typename traits::array_layout & layout , + // Impl::ViewAllocProp::size_type exists when the traits and allocation properties + // are valid for allocating viewed memory. + const typename Impl::ViewAllocProp< traits , AllocationProperties >::size_type = 0 ) + : m_ptr_on_device(0) + , m_offset_map() + , m_management() + { + typedef Impl::ViewAllocProp< traits , AllocationProperties > Alloc ; + + m_offset_map.assign( layout ); + if(Alloc::AllowPadding) + m_offset_map.set_padding(); + + m_ptr_on_device = view_data_management::template allocate< Alloc::Initialize >( Alloc::label(prop) , m_offset_map ); + + m_management.set_noncontiguous(); + } + + //------------------------------------ + // Assign an unmanaged View from pointer, can be called in functors. + // No alignment padding is performed. + + template< class Type > + explicit KOKKOS_INLINE_FUNCTION + View( Type * ptr , + typename Impl::ViewRawPointerProp< traits , Type >::size_type n0 = 0 , + const size_t n1 = 0 , + const size_t n2 = 0 , + const size_t n3 = 0 , + const size_t n4 = 0 , + const size_t n5 = 0 , + const size_t n6 = 0 , + const size_t n7 = 0 , + const size_t n8 = 0 ) + : m_ptr_on_device(ptr) + , m_offset_map() + , m_management() + { + m_offset_map.assign( n0, n1, n2, n3, n4, n5, n6, n7, n8 ); + m_management.set_unmanaged(); + } + + template< class Type > + explicit KOKKOS_INLINE_FUNCTION + View( Type * ptr , + typename traits::array_layout const & layout , + typename Impl::ViewRawPointerProp< traits , Type >::size_type = 0 ) + : m_ptr_on_device(ptr) + , m_offset_map() + , m_management() + { + m_offset_map.assign( layout ); + m_management.set_unmanaged(); + m_management.set_noncontiguous(); + } + + //------------------------------------ + /** \brief Constructors for subviews requires following + * type-compatibility condition, enforce via StaticAssert. + * + * Impl::is_same< View , + * typename Impl::ViewSubview< View + * , ArgType0 , ArgType1 , ArgType2 , ArgType3 + * , ArgType4 , ArgType5 , ArgType6 , ArgType7 + * >::type >::value + */ + template< class D , class A1 , class A2 , class A3 + , class SubArg0_type , class SubArg1_type , class SubArg2_type , class SubArg3_type + , class SubArg4_type , class SubArg5_type , class SubArg6_type , class SubArg7_type + > + KOKKOS_INLINE_FUNCTION + View( const View & src + , const SubArg0_type & arg0 , const SubArg1_type & arg1 + , const SubArg2_type & arg2 , const SubArg3_type & arg3 + , const SubArg4_type & arg4 , const SubArg5_type & arg5 + , const SubArg6_type & arg6 , const SubArg7_type & arg7 + ); + + template< class D , class A1 , class A2 , class A3 + , class SubArg0_type , class SubArg1_type , class SubArg2_type , class SubArg3_type + , class SubArg4_type , class SubArg5_type , class SubArg6_type + > + KOKKOS_INLINE_FUNCTION + View( const View & src + , const SubArg0_type & arg0 , const SubArg1_type & arg1 + , const SubArg2_type & arg2 , const SubArg3_type & arg3 + , const SubArg4_type & arg4 , const SubArg5_type & arg5 + , const SubArg6_type & arg6 + ); + + template< class D , class A1 , class A2 , class A3 + , class SubArg0_type , class SubArg1_type , class SubArg2_type , class SubArg3_type + , class SubArg4_type , class SubArg5_type + > + KOKKOS_INLINE_FUNCTION + View( const View & src + , const SubArg0_type & arg0 , const SubArg1_type & arg1 + , const SubArg2_type & arg2 , const SubArg3_type & arg3 + , const SubArg4_type & arg4 , const SubArg5_type & arg5 + ); + + template< class D , class A1 , class A2 , class A3 + , class SubArg0_type , class SubArg1_type , class SubArg2_type , class SubArg3_type + , class SubArg4_type + > + KOKKOS_INLINE_FUNCTION + View( const View & src + , const SubArg0_type & arg0 , const SubArg1_type & arg1 + , const SubArg2_type & arg2 , const SubArg3_type & arg3 + , const SubArg4_type & arg4 + ); + + template< class D , class A1 , class A2 , class A3 + , class SubArg0_type , class SubArg1_type , class SubArg2_type , class SubArg3_type + > + KOKKOS_INLINE_FUNCTION + View( const View & src + , const SubArg0_type & arg0 , const SubArg1_type & arg1 + , const SubArg2_type & arg2 , const SubArg3_type & arg3 + ); + + template< class D , class A1 , class A2 , class A3 + , class SubArg0_type , class SubArg1_type , class SubArg2_type + > + KOKKOS_INLINE_FUNCTION + View( const View & src + , const SubArg0_type & arg0 , const SubArg1_type & arg1 + , const SubArg2_type & arg2 + ); + + template< class D , class A1 , class A2 , class A3 + , class SubArg0_type , class SubArg1_type + > + KOKKOS_INLINE_FUNCTION + View( const View & src + , const SubArg0_type & arg0 , const SubArg1_type & arg1 + ); + + template< class D , class A1 , class A2 , class A3 + , class SubArg0_type + > + KOKKOS_INLINE_FUNCTION + View( const View & src + , const SubArg0_type & arg0 + ); + + //------------------------------------ + // Assign unmanaged View to portion of execution space's shared memory + + typedef Impl::if_c< ! traits::is_managed , + const typename traits::execution_space::scratch_memory_space & , + Impl::ViewError::device_shmem_constructor_requires_unmanaged > + if_scratch_memory_constructor ; + + explicit KOKKOS_INLINE_FUNCTION + View( typename if_scratch_memory_constructor::type space , + const unsigned n0 = 0 , + const unsigned n1 = 0 , + const unsigned n2 = 0 , + const unsigned n3 = 0 , + const unsigned n4 = 0 , + const unsigned n5 = 0 , + const unsigned n6 = 0 , + const unsigned n7 = 0 ) + : m_ptr_on_device(0) + , m_offset_map() + , m_management() + { + typedef typename traits::value_type value_type_ ; + + enum { align = 8 }; + enum { mask = align - 1 }; + + m_offset_map.assign( n0, n1, n2, n3, n4, n5, n6, n7 ); + + typedef Impl::if_c< ! traits::is_managed , + value_type_ * , + Impl::ViewError::device_shmem_constructor_requires_unmanaged > + if_device_shmem_pointer ; + + // Select the first argument: + m_ptr_on_device = if_device_shmem_pointer::select( + (value_type_*) space.get_shmem( unsigned( sizeof(value_type_) * m_offset_map.capacity() + unsigned(mask) ) & ~unsigned(mask) ) ); + } + + explicit KOKKOS_INLINE_FUNCTION + View( typename if_scratch_memory_constructor::type space , + typename traits::array_layout const & layout) + : m_ptr_on_device(0) + , m_offset_map() + , m_management() + { + typedef typename traits::value_type value_type_ ; + + typedef Impl::if_c< ! traits::is_managed , + value_type_ * , + Impl::ViewError::device_shmem_constructor_requires_unmanaged > + if_device_shmem_pointer ; + + m_offset_map.assign( layout ); + m_management.set_unmanaged(); + m_management.set_noncontiguous(); + + enum { align = 8 }; + enum { mask = align - 1 }; + + // Select the first argument: + m_ptr_on_device = if_device_shmem_pointer::select( + (value_type_*) space.get_shmem( unsigned( sizeof(value_type_) * m_offset_map.capacity() + unsigned(mask) ) & ~unsigned(mask) ) ); + } + + static inline + unsigned shmem_size( const unsigned n0 = 0 , + const unsigned n1 = 0 , + const unsigned n2 = 0 , + const unsigned n3 = 0 , + const unsigned n4 = 0 , + const unsigned n5 = 0 , + const unsigned n6 = 0 , + const unsigned n7 = 0 ) + { + enum { align = 8 }; + enum { mask = align - 1 }; + + typedef typename traits::value_type value_type_ ; + + offset_map_type offset_map ; + + offset_map.assign( n0, n1, n2, n3, n4, n5, n6, n7 ); + + return unsigned( sizeof(value_type_) * offset_map.capacity() + unsigned(mask) ) & ~unsigned(mask) ; + } + + //------------------------------------ + // Is not allocated + + KOKKOS_FORCEINLINE_FUNCTION + bool is_null() const { return 0 == ptr_on_device() ; } + + //------------------------------------ + // Operators for scalar (rank zero) views. + + typedef Impl::if_c< traits::rank == 0 , + typename traits::value_type , + Impl::ViewError::scalar_operator_called_from_non_scalar_view > + if_scalar_operator ; + + KOKKOS_INLINE_FUNCTION + const View & operator = ( const typename if_scalar_operator::type & rhs ) const + { + KOKKOS_RESTRICT_EXECUTION_TO_DATA( typename traits::memory_space , ptr_on_device() ); + *m_ptr_on_device = if_scalar_operator::select( rhs ); + return *this ; + } + + KOKKOS_FORCEINLINE_FUNCTION + operator typename if_scalar_operator::type & () const + { + KOKKOS_RESTRICT_EXECUTION_TO_DATA( typename traits::memory_space , ptr_on_device() ); + return if_scalar_operator::select( *m_ptr_on_device ); + } + + KOKKOS_FORCEINLINE_FUNCTION + typename if_scalar_operator::type & operator()() const + { + KOKKOS_RESTRICT_EXECUTION_TO_DATA( typename traits::memory_space , ptr_on_device() ); + return if_scalar_operator::select( *m_ptr_on_device ); + } + + KOKKOS_FORCEINLINE_FUNCTION + typename if_scalar_operator::type & operator*() const + { + KOKKOS_RESTRICT_EXECUTION_TO_DATA( typename traits::memory_space , ptr_on_device() ); + return if_scalar_operator::select( *m_ptr_on_device ); + } + + //------------------------------------ + // Array member access operators enabled if + // (1) a zero value of all argument types are compile-time comparable to zero + // (2) the rank matches the number of arguments + // (3) the memory space is valid for the access + //------------------------------------ + // rank 1: + + template< typename iType0 > + KOKKOS_FORCEINLINE_FUNCTION + typename Impl::ViewEnableArrayOper< reference_type , traits, typename traits::array_layout, 1, iType0 >::type + operator[] ( const iType0 & i0 ) const + { + KOKKOS_ASSERT_SHAPE_BOUNDS_1( m_offset_map, i0 ); + KOKKOS_RESTRICT_EXECUTION_TO_DATA( typename traits::memory_space , ptr_on_device() ); + + return m_ptr_on_device[ i0 ]; + } + + template< typename iType0 > + KOKKOS_FORCEINLINE_FUNCTION + typename Impl::ViewEnableArrayOper< reference_type , traits, typename traits::array_layout, 1, iType0 >::type + operator() ( const iType0 & i0 ) const + { + KOKKOS_ASSERT_SHAPE_BOUNDS_1( m_offset_map, i0 ); + KOKKOS_RESTRICT_EXECUTION_TO_DATA( typename traits::memory_space , ptr_on_device() ); + + return m_ptr_on_device[ i0 ]; + } + + template< typename iType0 > + KOKKOS_FORCEINLINE_FUNCTION + typename Impl::ViewEnableArrayOper< reference_type , traits, typename traits::array_layout, 1, iType0 >::type + at( const iType0 & i0 , const int , const int , const int , + const int , const int , const int , const int ) const + { + KOKKOS_ASSERT_SHAPE_BOUNDS_1( m_offset_map, i0 ); + KOKKOS_RESTRICT_EXECUTION_TO_DATA( typename traits::memory_space , ptr_on_device() ); + + return m_ptr_on_device[ i0 ]; + } + + // rank 2: + + template< typename iType0 , typename iType1 > + KOKKOS_FORCEINLINE_FUNCTION + typename Impl::ViewEnableArrayOper< reference_type , + traits, typename traits::array_layout, 2, iType0, iType1 >::type + operator() ( const iType0 & i0 , const iType1 & i1 ) const + { + KOKKOS_ASSERT_SHAPE_BOUNDS_2( m_offset_map, i0,i1 ); + KOKKOS_RESTRICT_EXECUTION_TO_DATA( typename traits::memory_space , ptr_on_device() ); + + return m_ptr_on_device[ m_offset_map(i0,i1) ]; + } + + template< typename iType0 , typename iType1 > + KOKKOS_FORCEINLINE_FUNCTION + typename Impl::ViewEnableArrayOper< reference_type , + traits, typename traits::array_layout, 2, iType0, iType1 >::type + at( const iType0 & i0 , const iType1 & i1 , const int , const int , + const int , const int , const int , const int ) const + { + KOKKOS_ASSERT_SHAPE_BOUNDS_2( m_offset_map, i0,i1 ); + KOKKOS_RESTRICT_EXECUTION_TO_DATA( typename traits::memory_space , ptr_on_device() ); + + return m_ptr_on_device[ m_offset_map(i0,i1) ]; + } + + // rank 3: + + template< typename iType0 , typename iType1 , typename iType2 > + KOKKOS_FORCEINLINE_FUNCTION + typename Impl::ViewEnableArrayOper< reference_type , + traits, typename traits::array_layout, 3, iType0, iType1, iType2 >::type + operator() ( const iType0 & i0 , const iType1 & i1 , const iType2 & i2 ) const + { + KOKKOS_ASSERT_SHAPE_BOUNDS_3( m_offset_map, i0,i1,i2 ); + KOKKOS_RESTRICT_EXECUTION_TO_DATA( typename traits::memory_space , ptr_on_device() ); + + return m_ptr_on_device[ m_offset_map(i0,i1,i2) ]; + } + + template< typename iType0 , typename iType1 , typename iType2 > + KOKKOS_FORCEINLINE_FUNCTION + typename Impl::ViewEnableArrayOper< reference_type , + traits, typename traits::array_layout, 3, iType0, iType1, iType2 >::type + at( const iType0 & i0 , const iType1 & i1 , const iType2 & i2 , const int , + const int , const int , const int , const int ) const + { + KOKKOS_ASSERT_SHAPE_BOUNDS_3( m_offset_map, i0,i1,i2 ); + KOKKOS_RESTRICT_EXECUTION_TO_DATA( typename traits::memory_space , ptr_on_device() ); + + return m_ptr_on_device[ m_offset_map(i0,i1,i2) ]; + } + + // rank 4: + + template< typename iType0 , typename iType1 , typename iType2 , typename iType3 > + KOKKOS_FORCEINLINE_FUNCTION + typename Impl::ViewEnableArrayOper< reference_type , + traits, typename traits::array_layout, 4, iType0, iType1, iType2, iType3 >::type + operator() ( const iType0 & i0 , const iType1 & i1 , const iType2 & i2 , const iType3 & i3 ) const + { + KOKKOS_ASSERT_SHAPE_BOUNDS_4( m_offset_map, i0,i1,i2,i3 ); + KOKKOS_RESTRICT_EXECUTION_TO_DATA( typename traits::memory_space , ptr_on_device() ); + + return m_ptr_on_device[ m_offset_map(i0,i1,i2,i3) ]; + } + + template< typename iType0 , typename iType1 , typename iType2 , typename iType3 > + KOKKOS_FORCEINLINE_FUNCTION + typename Impl::ViewEnableArrayOper< reference_type , + traits, typename traits::array_layout, 4, iType0, iType1, iType2, iType3 >::type + at( const iType0 & i0 , const iType1 & i1 , const iType2 & i2 , const iType3 & i3 , + const int , const int , const int , const int ) const + { + KOKKOS_ASSERT_SHAPE_BOUNDS_4( m_offset_map, i0,i1,i2,i3 ); + KOKKOS_RESTRICT_EXECUTION_TO_DATA( typename traits::memory_space , ptr_on_device() ); + + return m_ptr_on_device[ m_offset_map(i0,i1,i2,i3) ]; + } + + // rank 5: + + template< typename iType0 , typename iType1 , typename iType2 , typename iType3 , + typename iType4 > + KOKKOS_FORCEINLINE_FUNCTION + typename Impl::ViewEnableArrayOper< reference_type , + traits, typename traits::array_layout, 5, iType0, iType1, iType2, iType3 , iType4 >::type + operator() ( const iType0 & i0 , const iType1 & i1 , const iType2 & i2 , const iType3 & i3 , + const iType4 & i4 ) const + { + KOKKOS_ASSERT_SHAPE_BOUNDS_5( m_offset_map, i0,i1,i2,i3,i4 ); + KOKKOS_RESTRICT_EXECUTION_TO_DATA( typename traits::memory_space , ptr_on_device() ); + + return m_ptr_on_device[ m_offset_map(i0,i1,i2,i3,i4) ]; + } + + template< typename iType0 , typename iType1 , typename iType2 , typename iType3 , + typename iType4 > + KOKKOS_FORCEINLINE_FUNCTION + typename Impl::ViewEnableArrayOper< reference_type , + traits, typename traits::array_layout, 5, iType0, iType1, iType2, iType3 , iType4 >::type + at( const iType0 & i0 , const iType1 & i1 , const iType2 & i2 , const iType3 & i3 , + const iType4 & i4 , const int , const int , const int ) const + { + KOKKOS_ASSERT_SHAPE_BOUNDS_5( m_offset_map, i0,i1,i2,i3,i4 ); + KOKKOS_RESTRICT_EXECUTION_TO_DATA( typename traits::memory_space , ptr_on_device() ); + + return m_ptr_on_device[ m_offset_map(i0,i1,i2,i3,i4) ]; + } + + // rank 6: + + template< typename iType0 , typename iType1 , typename iType2 , typename iType3 , + typename iType4 , typename iType5 > + KOKKOS_FORCEINLINE_FUNCTION + typename Impl::ViewEnableArrayOper< reference_type , + traits, typename traits::array_layout, 6, + iType0, iType1, iType2, iType3 , iType4, iType5 >::type + operator() ( const iType0 & i0 , const iType1 & i1 , const iType2 & i2 , const iType3 & i3 , + const iType4 & i4 , const iType5 & i5 ) const + { + KOKKOS_ASSERT_SHAPE_BOUNDS_6( m_offset_map, i0,i1,i2,i3,i4,i5 ); + KOKKOS_RESTRICT_EXECUTION_TO_DATA( typename traits::memory_space , ptr_on_device() ); + + return m_ptr_on_device[ m_offset_map(i0,i1,i2,i3,i4,i5) ]; + } + + template< typename iType0 , typename iType1 , typename iType2 , typename iType3 , + typename iType4 , typename iType5 > + KOKKOS_FORCEINLINE_FUNCTION + typename Impl::ViewEnableArrayOper< reference_type , + traits, typename traits::array_layout, 6, + iType0, iType1, iType2, iType3 , iType4, iType5 >::type + at( const iType0 & i0 , const iType1 & i1 , const iType2 & i2 , const iType3 & i3 , + const iType4 & i4 , const iType5 & i5 , const int , const int ) const + { + KOKKOS_ASSERT_SHAPE_BOUNDS_6( m_offset_map, i0,i1,i2,i3,i4,i5 ); + KOKKOS_RESTRICT_EXECUTION_TO_DATA( typename traits::memory_space , ptr_on_device() ); + + return m_ptr_on_device[ m_offset_map(i0,i1,i2,i3,i4,i5) ]; + } + + // rank 7: + + template< typename iType0 , typename iType1 , typename iType2 , typename iType3 , + typename iType4 , typename iType5 , typename iType6 > + KOKKOS_FORCEINLINE_FUNCTION + typename Impl::ViewEnableArrayOper< reference_type , + traits, typename traits::array_layout, 7, + iType0, iType1, iType2, iType3 , iType4, iType5, iType6 >::type + operator() ( const iType0 & i0 , const iType1 & i1 , const iType2 & i2 , const iType3 & i3 , + const iType4 & i4 , const iType5 & i5 , const iType6 & i6 ) const + { + KOKKOS_ASSERT_SHAPE_BOUNDS_7( m_offset_map, i0,i1,i2,i3,i4,i5,i6 ); + KOKKOS_RESTRICT_EXECUTION_TO_DATA( typename traits::memory_space , ptr_on_device() ); + + return m_ptr_on_device[ m_offset_map(i0,i1,i2,i3,i4,i5,i6) ]; + } + + template< typename iType0 , typename iType1 , typename iType2 , typename iType3 , + typename iType4 , typename iType5 , typename iType6 > + KOKKOS_FORCEINLINE_FUNCTION + typename Impl::ViewEnableArrayOper< reference_type , + traits, typename traits::array_layout, 7, + iType0, iType1, iType2, iType3 , iType4, iType5, iType6 >::type + at( const iType0 & i0 , const iType1 & i1 , const iType2 & i2 , const iType3 & i3 , + const iType4 & i4 , const iType5 & i5 , const iType6 & i6 , const int ) const + { + KOKKOS_ASSERT_SHAPE_BOUNDS_7( m_offset_map, i0,i1,i2,i3,i4,i5,i6 ); + KOKKOS_RESTRICT_EXECUTION_TO_DATA( typename traits::memory_space , ptr_on_device() ); + + return m_ptr_on_device[ m_offset_map(i0,i1,i2,i3,i4,i5,i6) ]; + } + + // rank 8: + + template< typename iType0 , typename iType1 , typename iType2 , typename iType3 , + typename iType4 , typename iType5 , typename iType6 , typename iType7 > + KOKKOS_FORCEINLINE_FUNCTION + typename Impl::ViewEnableArrayOper< reference_type , + traits, typename traits::array_layout, 8, + iType0, iType1, iType2, iType3 , iType4, iType5, iType6, iType7 >::type + operator() ( const iType0 & i0 , const iType1 & i1 , const iType2 & i2 , const iType3 & i3 , + const iType4 & i4 , const iType5 & i5 , const iType6 & i6 , const iType7 & i7 ) const + { + KOKKOS_ASSERT_SHAPE_BOUNDS_8( m_offset_map, i0,i1,i2,i3,i4,i5,i6,i7 ); + KOKKOS_RESTRICT_EXECUTION_TO_DATA( typename traits::memory_space , ptr_on_device() ); + + return m_ptr_on_device[ m_offset_map(i0,i1,i2,i3,i4,i5,i6,i7) ]; + } + + template< typename iType0 , typename iType1 , typename iType2 , typename iType3 , + typename iType4 , typename iType5 , typename iType6 , typename iType7 > + KOKKOS_FORCEINLINE_FUNCTION + typename Impl::ViewEnableArrayOper< reference_type , + traits, typename traits::array_layout, 8, + iType0, iType1, iType2, iType3 , iType4, iType5, iType6, iType7 >::type + at( const iType0 & i0 , const iType1 & i1 , const iType2 & i2 , const iType3 & i3 , + const iType4 & i4 , const iType5 & i5 , const iType6 & i6 , const iType7 & i7 ) const + { + KOKKOS_ASSERT_SHAPE_BOUNDS_8( m_offset_map, i0,i1,i2,i3,i4,i5,i6,i7 ); + KOKKOS_RESTRICT_EXECUTION_TO_DATA( typename traits::memory_space , ptr_on_device() ); + + return m_ptr_on_device[ m_offset_map(i0,i1,i2,i3,i4,i5,i6,i7) ]; + } + + //------------------------------------ + // Access to the underlying contiguous storage of this view specialization. + // These methods are specific to specialization of a view. + + KOKKOS_FORCEINLINE_FUNCTION + typename traits::value_type * ptr_on_device() const + { return (typename traits::value_type *) m_ptr_on_device ; } + + // Stride of physical storage, dimensioned to at least Rank + template< typename iType > + KOKKOS_INLINE_FUNCTION + void stride( iType * const s ) const + { m_offset_map.stride(s); } + + // Count of contiguously allocated data members including padding. + KOKKOS_INLINE_FUNCTION + typename traits::size_type capacity() const + { return m_offset_map.capacity(); } + + // If the view data can be treated (deep copied) + // as a contiguous block of memory. + KOKKOS_INLINE_FUNCTION + bool is_contiguous() const + { return m_management.is_contiguous(); } +}; + +} /* namespace Kokkos */ + +//---------------------------------------------------------------------------- +//---------------------------------------------------------------------------- + +namespace Kokkos { + +template< class LT , class LL , class LD , class LM , class LS , + class RT , class RL , class RD , class RM , class RS > +KOKKOS_INLINE_FUNCTION +typename Impl::enable_if<( Impl::is_same< LS , RS >::value ), bool >::type +operator == ( const View & lhs , + const View & rhs ) +{ + // Same data, layout, dimensions + typedef ViewTraits lhs_traits ; + typedef ViewTraits rhs_traits ; + + return + Impl::is_same< typename lhs_traits::const_data_type , + typename rhs_traits::const_data_type >::value && + Impl::is_same< typename lhs_traits::array_layout , + typename rhs_traits::array_layout >::value && + Impl::is_same< typename lhs_traits::memory_space , + typename rhs_traits::memory_space >::value && + Impl::is_same< typename lhs_traits::specialize , + typename rhs_traits::specialize >::value && + lhs.ptr_on_device() == rhs.ptr_on_device() && + lhs.shape() == rhs.shape() ; +} + +template< class LT , class LL , class LD , class LM , class LS , + class RT , class RL , class RD , class RM , class RS > +KOKKOS_INLINE_FUNCTION +bool operator != ( const View & lhs , + const View & rhs ) +{ + return ! operator==( lhs , rhs ); +} + +//---------------------------------------------------------------------------- + + +} // namespace Kokkos + +//---------------------------------------------------------------------------- +//---------------------------------------------------------------------------- + +namespace Kokkos { + +//---------------------------------------------------------------------------- +/** \brief Deep copy a value into a view. + */ +template< class DT , class DL , class DD , class DM , class DS > +inline +void deep_copy( const View & dst , + typename Impl::enable_if<( + Impl::is_same< typename ViewTraits::non_const_value_type , + typename ViewTraits::value_type >::value + ), typename ViewTraits::const_value_type >::type & value ) +{ + Impl::ViewFill< View >( dst , value ); +} + +template< class ST , class SL , class SD , class SM , class SS > +inline +typename Impl::enable_if<( ViewTraits::rank == 0 )>::type +deep_copy( ST & dst , const View & src ) +{ + typedef ViewTraits src_traits ; + typedef typename src_traits::memory_space src_memory_space ; + Impl::DeepCopy< HostSpace , src_memory_space >( & dst , src.ptr_on_device() , sizeof(ST) ); +} + +//---------------------------------------------------------------------------- +/** \brief A deep copy between views of compatible type, and rank zero. + */ +template< class DT , class DL , class DD , class DM , class DS , + class ST , class SL , class SD , class SM , class SS > +inline +void deep_copy( const View & dst , + const View & src , + typename Impl::enable_if<( + // Same type and destination is not constant: + Impl::is_same< typename View::value_type , + typename View::non_const_value_type >::value + && + // Rank zero: + ( unsigned(View::rank) == unsigned(0) ) && + ( unsigned(View::rank) == unsigned(0) ) + )>::type * = 0 ) +{ + typedef View dst_type ; + typedef View src_type ; + + typedef typename dst_type::memory_space dst_memory_space ; + typedef typename src_type::memory_space src_memory_space ; + typedef typename src_type::value_type value_type ; + + if ( dst.ptr_on_device() != src.ptr_on_device() ) { + Impl::DeepCopy< dst_memory_space , src_memory_space >( dst.ptr_on_device() , src.ptr_on_device() , sizeof(value_type) ); + } +} + +//---------------------------------------------------------------------------- +/** \brief A deep copy between views of the default specialization, compatible type, + * same non-zero rank, same contiguous layout. + */ +template< class DT , class DL , class DD , class DM , + class ST , class SL , class SD , class SM > +inline +void deep_copy( const View & dst , + const View & src , + typename Impl::enable_if<( + // Same type and destination is not constant: + Impl::is_same< typename View::value_type , + typename View::non_const_value_type >::value + && + // Same non-zero rank: + ( unsigned(View::rank) == + unsigned(View::rank) ) + && + ( 0 < unsigned(View::rank) ) + && + // Same layout: + Impl::is_same< typename View::array_layout , + typename View::array_layout >::value + )>::type * = 0 ) +{ + typedef View dst_type ; + typedef View src_type ; + + typedef typename dst_type::memory_space dst_memory_space ; + typedef typename src_type::memory_space src_memory_space ; + + enum { is_contiguous = // Contiguous (e.g., non-strided, non-tiled) layout + Impl::is_same< typename View::array_layout , LayoutLeft >::value || + Impl::is_same< typename View::array_layout , LayoutRight >::value }; + + if ( dst.ptr_on_device() != src.ptr_on_device() ) { + + // Same shape (dimensions) + Impl::assert_shapes_are_equal( dst.shape() , src.shape() ); + + if ( is_contiguous && dst.capacity() == src.capacity() ) { + + // Views span equal length contiguous range. + // Assuming can perform a straight memory copy over this range. + + const size_t nbytes = sizeof(typename dst_type::value_type) * dst.capacity(); + + Impl::DeepCopy< dst_memory_space , src_memory_space >( dst.ptr_on_device() , src.ptr_on_device() , nbytes ); + } + else { + // Destination view's execution space must be able to directly access source memory space + // in order for the ViewRemap functor run in the destination memory space's execution space. + Impl::ViewRemap< dst_type , src_type >( dst , src ); + } + } +} + + +/** \brief Deep copy equal dimension arrays in the same space which + * have different layouts or specializations. + */ +template< class DT , class DL , class DD , class DM , class DS , + class ST , class SL , class SD , class SM , class SS > +inline +void deep_copy( const View< DT, DL, DD, DM, DS > & dst , + const View< ST, SL, SD, SM, SS > & src , + const typename Impl::enable_if<( + // Same type and destination is not constant: + Impl::is_same< typename View::value_type , + typename View::non_const_value_type >::value + && + // Source memory space is accessible to destination memory space + Impl::VerifyExecutionCanAccessMemorySpace< typename View::memory_space + , typename View::memory_space >::value + && + // Same non-zero rank + ( unsigned( View::rank ) == + unsigned( View::rank ) ) + && + ( 0 < unsigned( View::rank ) ) + && + // Different layout or different specialization: + ( ( ! Impl::is_same< typename View::array_layout , + typename View::array_layout >::value ) + || + ( ! Impl::is_same< DS , SS >::value ) + ) + )>::type * = 0 ) +{ + typedef View< DT, DL, DD, DM, DS > dst_type ; + typedef View< ST, SL, SD, SM, SS > src_type ; + + assert_shapes_equal_dimension( dst.shape() , src.shape() ); + + Impl::ViewRemap< dst_type , src_type >( dst , src ); +} + +//---------------------------------------------------------------------------- +//---------------------------------------------------------------------------- + +template< class T , class L , class D , class M , class S > +typename Impl::enable_if<( + View::is_managed + ), typename View::HostMirror >::type +inline +create_mirror( const View & src ) +{ + typedef View view_type ; + typedef typename view_type::HostMirror host_view_type ; + typedef typename view_type::memory_space memory_space ; + + // 'view' is managed therefore we can allocate a + // compatible host_view through the ordinary constructor. + + std::string label = memory_space::query_label( src.ptr_on_device() ); + label.append("_mirror"); + + return host_view_type( label , + src.dimension_0() , + src.dimension_1() , + src.dimension_2() , + src.dimension_3() , + src.dimension_4() , + src.dimension_5() , + src.dimension_6() , + src.dimension_7() ); +} + +template< class T , class L , class D , class M , class S > +typename Impl::enable_if<( + View::is_managed && + Impl::ViewAssignable< typename View::HostMirror , View >::value + ), typename View::HostMirror >::type +inline +create_mirror_view( const View & src ) +{ + return src ; +} + +template< class T , class L , class D , class M , class S > +typename Impl::enable_if<( + View::is_managed && + ! Impl::ViewAssignable< typename View::HostMirror , View >::value + ), typename View::HostMirror >::type +inline +create_mirror_view( const View & src ) +{ + return create_mirror( src ); +} + +//---------------------------------------------------------------------------- + +/** \brief Resize a view with copying old data to new data at the corresponding indices. */ +template< class T , class L , class D , class M , class S > +inline +void resize( View & v , + const typename Impl::enable_if< ViewTraits::is_managed , size_t >::type n0 , + const size_t n1 = 0 , + const size_t n2 = 0 , + const size_t n3 = 0 , + const size_t n4 = 0 , + const size_t n5 = 0 , + const size_t n6 = 0 , + const size_t n7 = 0 ) +{ + typedef View view_type ; + typedef typename view_type::memory_space memory_space ; + + const std::string label = memory_space::query_label( v.ptr_on_device() ); + + view_type v_resized( label, n0, n1, n2, n3, n4, n5, n6, n7 ); + + Impl::ViewRemap< view_type , view_type >( v_resized , v ); + + v = v_resized ; +} + +/** \brief Reallocate a view without copying old data to new data */ +template< class T , class L , class D , class M , class S > +inline +void realloc( View & v , + const typename Impl::enable_if< ViewTraits::is_managed , size_t >::type n0 , + const size_t n1 = 0 , + const size_t n2 = 0 , + const size_t n3 = 0 , + const size_t n4 = 0 , + const size_t n5 = 0 , + const size_t n6 = 0 , + const size_t n7 = 0 ) +{ + typedef View view_type ; + typedef typename view_type::memory_space memory_space ; + + // Query the current label and reuse it. + const std::string label = memory_space::query_label( v.ptr_on_device() ); + + v = view_type(); // deallocate first, if the only view to memory. + v = view_type( label, n0, n1, n2, n3, n4, n5, n6, n7 ); +} + +} // namespace Kokkos + +//---------------------------------------------------------------------------- +//---------------------------------------------------------------------------- + +namespace Kokkos { + +struct ALL { KOKKOS_INLINE_FUNCTION ALL(){} }; + +template< class DstViewType , + class T , class L , class D , class M , class S , + class ArgType0 > +KOKKOS_INLINE_FUNCTION +DstViewType +subview( const View & src , + const ArgType0 & arg0 ) +{ + DstViewType dst ; + + Impl::ViewAssignment( dst , src , arg0 ); + + return dst ; +} + +template< class DstViewType , + class T , class L , class D , class M , class S , + class ArgType0 , class ArgType1 > +KOKKOS_INLINE_FUNCTION +DstViewType +subview( const View & src , + const ArgType0 & arg0 , + const ArgType1 & arg1 ) +{ + DstViewType dst ; + + Impl::ViewAssignment( dst, src, arg0, arg1 ); + + return dst ; +} + +template< class DstViewType , + class T , class L , class D , class M , class S , + class ArgType0 , class ArgType1 , class ArgType2 > +KOKKOS_INLINE_FUNCTION +DstViewType +subview( const View & src , + const ArgType0 & arg0 , + const ArgType1 & arg1 , + const ArgType2 & arg2 ) +{ + DstViewType dst ; + + Impl::ViewAssignment( dst, src, arg0, arg1, arg2 ); + + return dst ; +} + +template< class DstViewType , + class T , class L , class D , class M , class S , + class ArgType0 , class ArgType1 , class ArgType2 , class ArgType3 > +KOKKOS_INLINE_FUNCTION +DstViewType +subview( const View & src , + const ArgType0 & arg0 , + const ArgType1 & arg1 , + const ArgType2 & arg2 , + const ArgType3 & arg3 ) +{ + DstViewType dst ; + + Impl::ViewAssignment( dst, src, arg0, arg1, arg2, arg3 ); + + return dst ; +} + +template< class DstViewType , + class T , class L , class D , class M , class S , + class ArgType0 , class ArgType1 , class ArgType2 , class ArgType3 , + class ArgType4 > +KOKKOS_INLINE_FUNCTION +DstViewType +subview( const View & src , + const ArgType0 & arg0 , + const ArgType1 & arg1 , + const ArgType2 & arg2 , + const ArgType3 & arg3 , + const ArgType4 & arg4 ) +{ + DstViewType dst ; + + Impl::ViewAssignment( dst, src, arg0, arg1, arg2, arg3, arg4 ); + + return dst ; +} + +template< class DstViewType , + class T , class L , class D , class M , class S , + class ArgType0 , class ArgType1 , class ArgType2 , class ArgType3 , + class ArgType4 , class ArgType5 > +KOKKOS_INLINE_FUNCTION +DstViewType +subview( const View & src , + const ArgType0 & arg0 , + const ArgType1 & arg1 , + const ArgType2 & arg2 , + const ArgType3 & arg3 , + const ArgType4 & arg4 , + const ArgType5 & arg5 ) +{ + DstViewType dst ; + + Impl::ViewAssignment( dst, src, arg0, arg1, arg2, arg3, arg4, arg5 ); + + return dst ; +} + +template< class DstViewType , + class T , class L , class D , class M , class S , + class ArgType0 , class ArgType1 , class ArgType2 , class ArgType3 , + class ArgType4 , class ArgType5 , class ArgType6 > +KOKKOS_INLINE_FUNCTION +DstViewType +subview( const View & src , + const ArgType0 & arg0 , + const ArgType1 & arg1 , + const ArgType2 & arg2 , + const ArgType3 & arg3 , + const ArgType4 & arg4 , + const ArgType5 & arg5 , + const ArgType6 & arg6 ) +{ + DstViewType dst ; + + Impl::ViewAssignment( dst, src, arg0, arg1, arg2, arg3, arg4, arg5, arg6 ); + + return dst ; +} + +template< class DstViewType , + class T , class L , class D , class M , class S , + class ArgType0 , class ArgType1 , class ArgType2 , class ArgType3 , + class ArgType4 , class ArgType5 , class ArgType6 , class ArgType7 > +KOKKOS_INLINE_FUNCTION +DstViewType +subview( const View & src , + const ArgType0 & arg0 , + const ArgType1 & arg1 , + const ArgType2 & arg2 , + const ArgType3 & arg3 , + const ArgType4 & arg4 , + const ArgType5 & arg5 , + const ArgType6 & arg6 , + const ArgType7 & arg7 ) +{ + DstViewType dst ; + + Impl::ViewAssignment( dst, src, arg0, arg1, arg2, arg3, arg4, arg5, arg6, arg7 ); + + return dst ; +} + +} // namespace Kokkos + +//---------------------------------------------------------------------------- + +namespace Kokkos { + +template< class D , class A1 , class A2 , class A3 , class S , + class ArgType0 , class ArgType1 , class ArgType2 , class ArgType3 , + class ArgType4 , class ArgType5 , class ArgType6 , class ArgType7 > +KOKKOS_INLINE_FUNCTION +typename Impl::ViewSubview< View + , ArgType0 , ArgType1 , ArgType2 , ArgType3 + , ArgType4 , ArgType5 , ArgType6 , ArgType7 + >::type +subview( const View & src , + const ArgType0 & arg0 , + const ArgType1 & arg1 , + const ArgType2 & arg2 , + const ArgType3 & arg3 , + const ArgType4 & arg4 , + const ArgType5 & arg5 , + const ArgType6 & arg6 , + const ArgType7 & arg7 ) +{ + typedef typename + Impl::ViewSubview< View + , ArgType0 , ArgType1 , ArgType2 , ArgType3 + , ArgType4 , ArgType5 , ArgType6 , ArgType7 + >::type + DstViewType ; + + return DstViewType( src, arg0, arg1, arg2, arg3, arg4, arg5, arg6, arg7 ); +} + +template< class D , class A1 , class A2 , class A3 , class S , + class ArgType0 , class ArgType1 , class ArgType2 , class ArgType3 , + class ArgType4 , class ArgType5 , class ArgType6 > +KOKKOS_INLINE_FUNCTION +typename Impl::ViewSubview< View + , ArgType0 , ArgType1 , ArgType2 , ArgType3 + , ArgType4 , ArgType5 , ArgType6 , void + >::type +subview( const View & src , + const ArgType0 & arg0 , + const ArgType1 & arg1 , + const ArgType2 & arg2 , + const ArgType3 & arg3 , + const ArgType4 & arg4 , + const ArgType5 & arg5 , + const ArgType6 & arg6 ) +{ + typedef typename + Impl::ViewSubview< View + , ArgType0 , ArgType1 , ArgType2 , ArgType3 + , ArgType4 , ArgType5 , ArgType6 , void + >::type + DstViewType ; + + return DstViewType( src, arg0, arg1, arg2, arg3, arg4, arg5, arg6 ); +} + +template< class D , class A1 , class A2 , class A3 , class S , + class ArgType0 , class ArgType1 , class ArgType2 , class ArgType3 , + class ArgType4 , class ArgType5 > +KOKKOS_INLINE_FUNCTION +typename Impl::ViewSubview< View + , ArgType0 , ArgType1 , ArgType2 , ArgType3 + , ArgType4 , ArgType5 , void , void + >::type +subview( const View & src , + const ArgType0 & arg0 , + const ArgType1 & arg1 , + const ArgType2 & arg2 , + const ArgType3 & arg3 , + const ArgType4 & arg4 , + const ArgType5 & arg5 ) +{ + typedef typename + Impl::ViewSubview< View + , ArgType0 , ArgType1 , ArgType2 , ArgType3 + , ArgType4 , ArgType5 , void , void + >::type + DstViewType ; + + return DstViewType( src, arg0, arg1, arg2, arg3, arg4, arg5 ); +} + +template< class D , class A1 , class A2 , class A3 , class S , + class ArgType0 , class ArgType1 , class ArgType2 , class ArgType3 , + class ArgType4 > +KOKKOS_INLINE_FUNCTION +typename Impl::ViewSubview< View + , ArgType0 , ArgType1 , ArgType2 , ArgType3 + , ArgType4 , void , void , void + >::type +subview( const View & src , + const ArgType0 & arg0 , + const ArgType1 & arg1 , + const ArgType2 & arg2 , + const ArgType3 & arg3 , + const ArgType4 & arg4 ) +{ + typedef typename + Impl::ViewSubview< View + , ArgType0 , ArgType1 , ArgType2 , ArgType3 + , ArgType4 , void , void , void + >::type + DstViewType ; + + return DstViewType( src, arg0, arg1, arg2, arg3, arg4 ); +} + +template< class D , class A1 , class A2 , class A3 , class S , + class ArgType0 , class ArgType1 , class ArgType2 , class ArgType3 > +KOKKOS_INLINE_FUNCTION +typename Impl::ViewSubview< View + , ArgType0 , ArgType1 , ArgType2 , ArgType3 + , void , void , void , void + >::type +subview( const View & src , + const ArgType0 & arg0 , + const ArgType1 & arg1 , + const ArgType2 & arg2 , + const ArgType3 & arg3 ) +{ + typedef typename + Impl::ViewSubview< View + , ArgType0 , ArgType1 , ArgType2 , ArgType3 + , void , void , void , void + >::type + DstViewType ; + + return DstViewType( src, arg0, arg1, arg2, arg3 ); +} + +template< class D , class A1 , class A2 , class A3 , class S , + class ArgType0 , class ArgType1 , class ArgType2 > +KOKKOS_INLINE_FUNCTION +typename Impl::ViewSubview< View + , ArgType0 , ArgType1 , ArgType2 , void + , void , void , void , void + >::type +subview( const View & src , + const ArgType0 & arg0 , + const ArgType1 & arg1 , + const ArgType2 & arg2 ) +{ + typedef typename + Impl::ViewSubview< View + , ArgType0 , ArgType1 , ArgType2 , void + , void , void , void , void + >::type + DstViewType ; + + return DstViewType( src, arg0, arg1, arg2 ); +} + +template< class D , class A1 , class A2 , class A3 , class S , + class ArgType0 , class ArgType1 > +KOKKOS_INLINE_FUNCTION +typename Impl::ViewSubview< View + , ArgType0 , ArgType1 , void , void + , void , void , void , void + >::type +subview( const View & src , + const ArgType0 & arg0 , + const ArgType1 & arg1 ) +{ + typedef typename + Impl::ViewSubview< View + , ArgType0 , ArgType1 , void , void + , void , void , void , void + >::type + DstViewType ; + + return DstViewType( src, arg0, arg1 ); +} + +template< class D , class A1 , class A2 , class A3 , class S , + class ArgType0 > +KOKKOS_INLINE_FUNCTION +typename Impl::ViewSubview< View + , ArgType0 , void , void , void + , void , void , void , void + >::type +subview( const View & src , + const ArgType0 & arg0 ) +{ + typedef typename + Impl::ViewSubview< View + , ArgType0 , void , void , void + , void , void , void , void + >::type + DstViewType ; + + return DstViewType( src, arg0 ); +} + +} // namespace Kokkos + +//---------------------------------------------------------------------------- +//---------------------------------------------------------------------------- + +#include +#include + +//---------------------------------------------------------------------------- +//---------------------------------------------------------------------------- + +#endif + diff --git a/lib/kokkos/core/src/Kokkos_hwloc.hpp b/lib/kokkos/core/src/Kokkos_hwloc.hpp new file mode 100755 index 0000000000..6b8aea148d --- /dev/null +++ b/lib/kokkos/core/src/Kokkos_hwloc.hpp @@ -0,0 +1,140 @@ +/* +//@HEADER +// ************************************************************************ +// +// Kokkos: Manycore Performance-Portable Multidimensional Arrays +// Copyright (2012) Sandia Corporation +// +// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +// the U.S. Government retains certain rights in this software. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// 1. Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// 3. Neither the name of the Corporation nor the names of the +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Questions? Contact H. Carter Edwards (hcedwar@sandia.gov) +// +// ************************************************************************ +//@HEADER +*/ + +#ifndef KOKKOS_HWLOC_HPP +#define KOKKOS_HWLOC_HPP + +#include + +namespace Kokkos { + +/** \brief Minimal subset of logical 'hwloc' functionality available + * from http://www.open-mpi.org/projects/hwloc/. + * + * The calls are NOT thread safe in order to avoid mutexes, + * memory allocations, or other actions which could give the + * runtime system an opportunity to migrate the threads or + * touch allocated memory during the function calls. + * + * All calls to these functions should be performed by a thread + * when it has guaranteed exclusive access; e.g., for OpenMP + * within a 'critical' region. + */ +namespace hwloc { + +/** \brief Query if hwloc is available */ +bool available(); + +/** \brief Query number of available NUMA regions. + * This will be less than the hardware capacity + * if the MPI process is pinned to a NUMA region. + */ +unsigned get_available_numa_count(); + +/** \brief Query number of available cores per NUMA regions. + * This will be less than the hardware capacity + * if the MPI process is pinned to a set of cores. + */ +unsigned get_available_cores_per_numa(); + +/** \brief Query number of available "hard" threads per core; i.e., hyperthreads */ +unsigned get_available_threads_per_core(); + +} /* namespace hwloc */ +} /* namespace Kokkos */ + +//---------------------------------------------------------------------------- +//---------------------------------------------------------------------------- +// Internal functions for binding persistent spawned threads. + +namespace Kokkos { +namespace hwloc { + +/** \brief Recommend mapping of threads onto cores. + * + * If thread_count == 0 then choose and set a value. + * If use_numa_count == 0 then choose and set a value. + * If use_cores_per_numa == 0 then choose and set a value. + * + * Return 0 if asynchronous, + * Return 1 if synchronous and threads_coord[0] is process core + */ +unsigned thread_mapping( const char * const label , + const bool allow_async , + unsigned & thread_count , + unsigned & use_numa_count , + unsigned & use_cores_per_numa , + std::pair threads_coord[] ); + +/** \brief Query core-coordinate of the current thread + * with respect to the core_topology. + * + * As long as the thread is running within the + * process binding the following condition holds. + * + * core_coordinate.first < core_topology.first + * core_coordinate.second < core_topology.second + */ +std::pair get_this_thread_coordinate(); + +/** \brief Bind the current thread to a core. */ +bool bind_this_thread( const std::pair ); + +/** \brief Bind the current thread to one of the cores in the list. + * Set that entry to (~0,~0) and return the index. + * If binding fails return ~0. + */ +unsigned bind_this_thread( const unsigned coordinate_count , + std::pair coordinate[] ); + +/** \brief Unbind the current thread back to the original process binding */ +bool unbind_this_thread(); + +} /* namespace hwloc */ +} /* namespace Kokkos */ + +//---------------------------------------------------------------------------- +//---------------------------------------------------------------------------- + +#endif /* #define KOKKOS_HWLOC_HPP */ + diff --git a/lib/kokkos/core/src/OpenMP/Kokkos_OpenMP_Parallel.hpp b/lib/kokkos/core/src/OpenMP/Kokkos_OpenMP_Parallel.hpp new file mode 100755 index 0000000000..7c338e5858 --- /dev/null +++ b/lib/kokkos/core/src/OpenMP/Kokkos_OpenMP_Parallel.hpp @@ -0,0 +1,496 @@ +/* +//@HEADER +// ************************************************************************ +// +// Kokkos: Manycore Performance-Portable Multidimensional Arrays +// Copyright (2012) Sandia Corporation +// +// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +// the U.S. Government retains certain rights in this software. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// 1. Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// 3. Neither the name of the Corporation nor the names of the +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Questions? Contact H. Carter Edwards (hcedwar@sandia.gov) +// +// ************************************************************************ +//@HEADER +*/ + +#ifndef KOKKOS_OPENMP_PARALLEL_HPP +#define KOKKOS_OPENMP_PARALLEL_HPP + +#include + +#include +#include +#include + +//---------------------------------------------------------------------------- +//---------------------------------------------------------------------------- + +namespace Kokkos { +namespace Impl { + +template< class FunctorType , class Arg0 , class Arg1 , class Arg2 > +class ParallelFor< FunctorType , Kokkos::RangePolicy< Arg0 , Arg1 , Arg2 , Kokkos::OpenMP > > +{ +private: + + typedef Kokkos::RangePolicy< Arg0 , Arg1 , Arg2 , Kokkos::OpenMP > Policy ; + + template< class PType > + KOKKOS_FORCEINLINE_FUNCTION static + void driver( typename Impl::enable_if< Impl::is_same< typename PType::work_tag , void >::value , + const FunctorType & >::type functor + , const PType & range ) + { + const typename PType::member_type work_end = range.end(); + for ( typename PType::member_type iwork = range.begin() ; iwork < work_end ; ++iwork ) { + functor( iwork ); + } + } + + template< class PType > + KOKKOS_FORCEINLINE_FUNCTION static + void driver( typename Impl::enable_if< ! Impl::is_same< typename PType::work_tag , void >::value , + const FunctorType & >::type functor + , const PType & range ) + { + const typename PType::member_type work_end = range.end(); + for ( typename PType::member_type iwork = range.begin() ; iwork < work_end ; ++iwork ) { + functor( typename PType::work_tag() , iwork ); + } + } + +public: + + inline + ParallelFor( const FunctorType & functor + , const Policy & policy ) + { + OpenMPexec::verify_is_process("Kokkos::OpenMP parallel_for"); + OpenMPexec::verify_initialized("Kokkos::OpenMP parallel_for"); + +#pragma omp parallel + { + OpenMPexec & exec = * OpenMPexec::get_thread_omp(); + driver( functor , typename Policy::WorkRange( policy , exec.pool_rank() , exec.pool_size() ) ); + } +/* END #pragma omp parallel */ + } +}; + +} // namespace Impl +} // namespace Kokkos + +//---------------------------------------------------------------------------- +//---------------------------------------------------------------------------- + +namespace Kokkos { +namespace Impl { + +template< class FunctorType , class Arg0 , class Arg1 , class Arg2 > +class ParallelReduce< FunctorType , Kokkos::RangePolicy< Arg0 , Arg1 , Arg2 , Kokkos::OpenMP > > +{ +private: + + typedef Kokkos::RangePolicy< Arg0 , Arg1 , Arg2 , Kokkos::OpenMP > Policy ; + typedef typename Policy::work_tag WorkTag ; + typedef Kokkos::Impl::FunctorValueTraits< FunctorType , WorkTag > ValueTraits ; + typedef Kokkos::Impl::FunctorValueInit< FunctorType , WorkTag > ValueInit ; + typedef Kokkos::Impl::FunctorValueJoin< FunctorType , WorkTag > ValueJoin ; + + typedef typename ValueTraits::pointer_type pointer_type ; + typedef typename ValueTraits::reference_type reference_type ; + + template< class PType > + KOKKOS_FORCEINLINE_FUNCTION static + void driver( typename Impl::enable_if< Impl::is_same< typename PType::work_tag , void >::value , + const FunctorType & >::type functor + , reference_type update + , const PType & range ) + { + const typename PType::member_type work_end = range.end(); + for ( typename PType::member_type iwork = range.begin() ; iwork < work_end ; ++iwork ) { + functor( iwork , update ); + } + } + + template< class PType > + KOKKOS_FORCEINLINE_FUNCTION static + void driver( typename Impl::enable_if< ! Impl::is_same< typename PType::work_tag , void >::value , + const FunctorType & >::type functor + , reference_type update + , const PType & range ) + { + const typename PType::member_type work_end = range.end(); + for ( typename PType::member_type iwork = range.begin() ; iwork < work_end ; ++iwork ) { + functor( typename PType::work_tag() , iwork , update ); + } + } + +public: + + //---------------------------------------- + + template< class ViewType > + inline + ParallelReduce( typename Impl::enable_if< + ( Impl::is_view< ViewType >::value && + Impl::is_same< typename ViewType::memory_space , HostSpace >::value + ), const FunctorType & >::type functor + , const Policy & policy + , const ViewType & result_view ) + { + OpenMPexec::verify_is_process("Kokkos::OpenMP parallel_reduce"); + OpenMPexec::verify_initialized("Kokkos::OpenMP parallel_reduce"); + + OpenMPexec::resize_scratch( ValueTraits::value_size( functor ) , 0 ); + +#pragma omp parallel + { + OpenMPexec & exec = * OpenMPexec::get_thread_omp(); + + driver( functor + , ValueInit::init( functor , exec.scratch_reduce() ) + , typename Policy::WorkRange( policy , exec.pool_rank() , exec.pool_size() ) + ); + } +/* END #pragma omp parallel */ + + { + const pointer_type ptr = pointer_type( OpenMPexec::pool_rev(0)->scratch_reduce() ); + + for ( int i = 1 ; i < OpenMPexec::pool_size() ; ++i ) { + ValueJoin::join( functor , ptr , OpenMPexec::pool_rev(i)->scratch_reduce() ); + } + + Kokkos::Impl::FunctorFinal< FunctorType , WorkTag >::final( functor , ptr ); + + if ( result_view.ptr_on_device() ) { + const int n = ValueTraits::value_count( functor ); + + for ( int j = 0 ; j < n ; ++j ) { result_view.ptr_on_device()[j] = ptr[j] ; } + } + } + } +}; + +} // namespace Impl +} // namespace Kokkos + +//---------------------------------------------------------------------------- +//---------------------------------------------------------------------------- + +namespace Kokkos { +namespace Impl { + +template< class FunctorType , class Arg0 , class Arg1 , class Arg2 > +class ParallelScan< FunctorType , Kokkos::RangePolicy< Arg0 , Arg1 , Arg2 , Kokkos::OpenMP > > +{ +private: + + typedef Kokkos::RangePolicy< Arg0 , Arg1 , Arg2 , Kokkos::OpenMP > Policy ; + typedef typename Policy::work_tag WorkTag ; + typedef Kokkos::Impl::FunctorValueTraits< FunctorType , WorkTag > ValueTraits ; + typedef Kokkos::Impl::FunctorValueInit< FunctorType , WorkTag > ValueInit ; + typedef Kokkos::Impl::FunctorValueJoin< FunctorType , WorkTag > ValueJoin ; + typedef Kokkos::Impl::FunctorValueOps< FunctorType , WorkTag > ValueOps ; + + typedef typename ValueTraits::pointer_type pointer_type ; + typedef typename ValueTraits::reference_type reference_type ; + + template< class PType > + KOKKOS_FORCEINLINE_FUNCTION static + void driver( typename Impl::enable_if< Impl::is_same< typename PType::work_tag , void >::value , + const FunctorType & >::type functor + , reference_type update + , const PType & range + , const bool final ) + { + const typename PType::member_type work_end = range.end(); + for ( typename PType::member_type iwork = range.begin() ; iwork < work_end ; ++iwork ) { + functor( iwork , update , final ); + } + } + + template< class PType > + KOKKOS_FORCEINLINE_FUNCTION static + void driver( typename Impl::enable_if< ! Impl::is_same< typename PType::work_tag , void >::value , + const FunctorType & >::type functor + , reference_type update + , const PType & range + , const bool final ) + { + const typename PType::member_type work_end = range.end(); + for ( typename PType::member_type iwork = range.begin() ; iwork < work_end ; ++iwork ) { + functor( typename PType::work_tag() , iwork , update , final ); + } + } + +public: + + //---------------------------------------- + + inline + ParallelScan( const FunctorType & functor + , const Policy & policy ) + { + OpenMPexec::verify_is_process("Kokkos::OpenMP parallel_scan"); + OpenMPexec::verify_initialized("Kokkos::OpenMP parallel_scan"); + + OpenMPexec::resize_scratch( 2 * ValueTraits::value_size( functor ) , 0 ); + +#pragma omp parallel + { + OpenMPexec & exec = * OpenMPexec::get_thread_omp(); + + driver( functor + , ValueInit::init( functor , pointer_type( exec.scratch_reduce() ) + ValueTraits::value_count( functor ) ) + , typename Policy::WorkRange( policy , exec.pool_rank() , exec.pool_size() ) + , false ); + } +/* END #pragma omp parallel */ + + { + const unsigned thread_count = OpenMPexec::pool_size(); + const unsigned value_count = ValueTraits::value_count( functor ); + + pointer_type ptr_prev = 0 ; + + for ( unsigned rank_rev = thread_count ; rank_rev-- ; ) { + + pointer_type ptr = pointer_type( OpenMPexec::pool_rev(rank_rev)->scratch_reduce() ); + + if ( ptr_prev ) { + for ( unsigned i = 0 ; i < value_count ; ++i ) { ptr[i] = ptr_prev[ i + value_count ] ; } + ValueJoin::join( functor , ptr + value_count , ptr ); + } + else { + ValueInit::init( functor , ptr ); + } + + ptr_prev = ptr ; + } + } + +#pragma omp parallel + { + OpenMPexec & exec = * OpenMPexec::get_thread_omp(); + + driver( functor + , ValueOps::reference( pointer_type( exec.scratch_reduce() ) ) + , typename Policy::WorkRange( policy , exec.pool_rank() , exec.pool_size() ) + , true ); + } +/* END #pragma omp parallel */ + + } + + //---------------------------------------- +}; + +} // namespace Impl +} // namespace Kokkos + +//---------------------------------------------------------------------------- +//---------------------------------------------------------------------------- + +namespace Kokkos { +namespace Impl { + +template< class FunctorType , class Arg0 , class Arg1 > +class ParallelFor< FunctorType , Kokkos::TeamPolicy< Arg0 , Arg1 , Kokkos::OpenMP > > +{ +private: + + typedef Kokkos::TeamPolicy< Arg0 , Arg1 , Kokkos::OpenMP > Policy ; + + template< class TagType > + KOKKOS_FORCEINLINE_FUNCTION static + void driver( typename Impl::enable_if< Impl::is_same< TagType , void >::value , + const FunctorType & >::type functor + , const typename Policy::member_type & member ) + { functor( member ); } + + template< class TagType > + KOKKOS_FORCEINLINE_FUNCTION static + void driver( typename Impl::enable_if< ! Impl::is_same< TagType , void >::value , + const FunctorType & >::type functor + , const typename Policy::member_type & member ) + { functor( TagType() , member ); } + +public: + + inline + ParallelFor( const FunctorType & functor , + const Policy & policy ) + { + OpenMPexec::verify_is_process("Kokkos::OpenMP parallel_for"); + OpenMPexec::verify_initialized("Kokkos::OpenMP parallel_for"); + + const size_t team_reduce_size = Policy::member_type::team_reduce_size(); + const size_t team_shmem_size = FunctorTeamShmemSize< FunctorType >::value( functor , policy.team_size() ); + + OpenMPexec::resize_scratch( 0 , team_reduce_size + team_shmem_size ); + +#pragma omp parallel + { + typename Policy::member_type member( * OpenMPexec::get_thread_omp() , policy , team_shmem_size ); + + for ( ; member.valid() ; member.next() ) { + ParallelFor::template driver< typename Policy::work_tag >( functor , member ); + } + } +/* END #pragma omp parallel */ + } + + void wait() {} +}; + + +template< class FunctorType , class Arg0 , class Arg1 > +class ParallelReduce< FunctorType , Kokkos::TeamPolicy< Arg0 , Arg1 , Kokkos::OpenMP > > +{ +private: + + typedef Kokkos::TeamPolicy< Arg0 , Arg1 , Kokkos::OpenMP > Policy ; + typedef typename Policy::work_tag WorkTag ; + typedef Kokkos::Impl::FunctorValueTraits< FunctorType , WorkTag > ValueTraits ; + typedef Kokkos::Impl::FunctorValueInit< FunctorType , WorkTag > ValueInit ; + typedef Kokkos::Impl::FunctorValueJoin< FunctorType , WorkTag > ValueJoin ; + + typedef typename ValueTraits::pointer_type pointer_type ; + typedef typename ValueTraits::reference_type reference_type ; + + + template< class PType > + KOKKOS_FORCEINLINE_FUNCTION static + void driver( typename Impl::enable_if< Impl::is_same< typename PType::work_tag , void >::value , + const FunctorType & >::type functor + , const typename PType::member_type & member + , reference_type update ) + { functor( member , update ); } + + template< class PType > + KOKKOS_FORCEINLINE_FUNCTION static + void driver( typename Impl::enable_if< ! Impl::is_same< typename PType::work_tag , void >::value , + const FunctorType & >::type functor + , const typename PType::member_type & member + , reference_type update ) + { functor( typename PType::work_tag() , member , update ); } + +public: + + inline + ParallelReduce( const FunctorType & functor , + const Policy & policy ) + { + OpenMPexec::verify_is_process("Kokkos::OpenMP parallel_reduce"); + + const size_t team_reduce_size = Policy::member_type::team_reduce_size(); + const size_t team_shmem_size = FunctorTeamShmemSize< FunctorType >::value( functor , policy.team_size() ); + + OpenMPexec::resize_scratch( ValueTraits::value_size( functor ) , team_reduce_size + team_shmem_size ); + +#pragma omp parallel + { + OpenMPexec & exec = * OpenMPexec::get_thread_omp(); + + reference_type update = ValueInit::init( functor , exec.scratch_reduce() ); + + for ( typename Policy::member_type member( exec , policy , team_shmem_size ); member.valid() ; member.next() ) { + ParallelReduce::template driver< Policy >( functor , member , update ); + } + } +/* END #pragma omp parallel */ + + { + typedef Kokkos::Impl::FunctorValueJoin< FunctorType , WorkTag , reference_type > Join ; + + const pointer_type ptr = pointer_type( OpenMPexec::pool_rev(0)->scratch_reduce() ); + + for ( int i = 1 ; i < OpenMPexec::pool_size() ; ++i ) { + Join::join( functor , ptr , OpenMPexec::pool_rev(i)->scratch_reduce() ); + } + + Kokkos::Impl::FunctorFinal< FunctorType , WorkTag >::final( functor , ptr ); + } + } + + template< class ViewType > + inline + ParallelReduce( const FunctorType & functor , + const Policy & policy , + const ViewType & result ) + { + OpenMPexec::verify_is_process("Kokkos::OpenMP parallel_reduce"); + + const size_t team_reduce_size = Policy::member_type::team_reduce_size(); + const size_t team_shmem_size = FunctorTeamShmemSize< FunctorType >::value( functor , policy.team_size() ); + + OpenMPexec::resize_scratch( ValueTraits::value_size( functor ) , team_reduce_size + team_shmem_size ); + +#pragma omp parallel + { + OpenMPexec & exec = * OpenMPexec::get_thread_omp(); + + reference_type update = ValueInit::init( functor , exec.scratch_reduce() ); + + for ( typename Policy::member_type member( exec , policy , team_shmem_size ); member.valid() ; member.next() ) { + ParallelReduce::template driver< Policy >( functor , member , update ); + } + } +/* END #pragma omp parallel */ + + { + const pointer_type ptr = pointer_type( OpenMPexec::pool_rev(0)->scratch_reduce() ); + + for ( int i = 1 ; i < OpenMPexec::pool_size() ; ++i ) { + ValueJoin::join( functor , ptr , OpenMPexec::pool_rev(i)->scratch_reduce() ); + } + + Kokkos::Impl::FunctorFinal< FunctorType , WorkTag >::final( functor , ptr ); + + const int n = ValueTraits::value_count( functor ); + + for ( int j = 0 ; j < n ; ++j ) { result.ptr_on_device()[j] = ptr[j] ; } + } + } + + void wait() {} +}; + +} // namespace Impl +} // namespace Kokkos + +//---------------------------------------------------------------------------- +//---------------------------------------------------------------------------- + +#endif /* KOKKOS_OPENMP_PARALLEL_HPP */ + diff --git a/lib/kokkos/core/src/OpenMP/Kokkos_OpenMPexec.cpp b/lib/kokkos/core/src/OpenMP/Kokkos_OpenMPexec.cpp new file mode 100755 index 0000000000..25683182d4 --- /dev/null +++ b/lib/kokkos/core/src/OpenMP/Kokkos_OpenMPexec.cpp @@ -0,0 +1,365 @@ +/* +//@HEADER +// ************************************************************************ +// +// Kokkos: Manycore Performance-Portable Multidimensional Arrays +// Copyright (2012) Sandia Corporation +// +// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +// the U.S. Government retains certain rights in this software. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// 1. Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// 3. Neither the name of the Corporation nor the names of the +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Questions? Contact H. Carter Edwards (hcedwar@sandia.gov) +// +// ************************************************************************ +//@HEADER +*/ + +#include +#include +#include +#include +#include +#include +#include + +#ifdef KOKKOS_HAVE_OPENMP + +namespace Kokkos { +namespace Impl { +namespace { + +KOKKOS_INLINE_FUNCTION +int kokkos_omp_in_parallel(); + +int kokkos_omp_in_critical_region = ( Kokkos::HostSpace::register_in_parallel( kokkos_omp_in_parallel ) , 0 ); + +KOKKOS_INLINE_FUNCTION +int kokkos_omp_in_parallel() +{ +#ifndef __CUDA_ARCH__ + return omp_in_parallel() && ! kokkos_omp_in_critical_region ; +#else + return 0; +#endif +} + +bool s_using_hwloc = false; + +} // namespace +} // namespace Impl +} // namespace Kokkos + + +namespace Kokkos { +namespace Impl { + +int OpenMPexec::m_map_rank[ OpenMPexec::MAX_THREAD_COUNT ] = { 0 }; + +int OpenMPexec::m_pool_topo[ 4 ] = { 0 }; + +OpenMPexec * OpenMPexec::m_pool[ OpenMPexec::MAX_THREAD_COUNT ] = { 0 }; + +void OpenMPexec::verify_is_process( const char * const label ) +{ + if ( omp_in_parallel() ) { + std::string msg( label ); + msg.append( " ERROR: in parallel" ); + Kokkos::Impl::throw_runtime_exception( msg ); + } +} + +void OpenMPexec::verify_initialized( const char * const label ) +{ + if ( 0 == m_pool[0] ) { + std::string msg( label ); + msg.append( " ERROR: not initialized" ); + Kokkos::Impl::throw_runtime_exception( msg ); + } +} + +void OpenMPexec::clear_scratch() +{ +#pragma omp parallel + { + const int rank_rev = m_map_rank[ omp_get_thread_num() ]; + +#pragma omp critical + { + kokkos_omp_in_critical_region = 1 ; + + m_pool[ rank_rev ]->~OpenMPexec(); + HostSpace::decrement( m_pool[ rank_rev ] ); + m_pool[ rank_rev ] = 0 ; + + kokkos_omp_in_critical_region = 0 ; + } +/* END #pragma omp critical */ + } +/* END #pragma omp parallel */ +} + +void OpenMPexec::resize_scratch( size_t reduce_size , size_t thread_size ) +{ + enum { ALIGN_MASK = Kokkos::Impl::MEMORY_ALIGNMENT - 1 }; + enum { ALLOC_EXEC = ( sizeof(OpenMPexec) + ALIGN_MASK ) & ~ALIGN_MASK }; + + const size_t old_reduce_size = m_pool[0] ? m_pool[0]->m_scratch_reduce_end : 0 ; + const size_t old_thread_size = m_pool[0] ? m_pool[0]->m_scratch_thread_end - m_pool[0]->m_scratch_reduce_end : 0 ; + + reduce_size = ( reduce_size + ALIGN_MASK ) & ~ALIGN_MASK ; + thread_size = ( thread_size + ALIGN_MASK ) & ~ALIGN_MASK ; + + // Requesting allocation and old allocation is too small: + + const bool allocate = ( old_reduce_size < reduce_size ) || + ( old_thread_size < thread_size ); + + if ( allocate ) { + if ( reduce_size < old_reduce_size ) { reduce_size = old_reduce_size ; } + if ( thread_size < old_thread_size ) { thread_size = old_thread_size ; } + } + + const size_t alloc_size = allocate ? ALLOC_EXEC + reduce_size + thread_size : 0 ; + const int pool_size = m_pool_topo[0] ; + + if ( allocate ) { + + clear_scratch(); + +#pragma omp parallel + { + const int rank_rev = m_map_rank[ omp_get_thread_num() ]; + const int rank = pool_size - ( rank_rev + 1 ); + +#pragma omp critical + { + kokkos_omp_in_critical_region = 1 ; + + m_pool[ rank_rev ] = + (OpenMPexec *) HostSpace::allocate( "openmp_scratch" , alloc_size ); + new( m_pool[ rank_rev ] ) OpenMPexec( rank , ALLOC_EXEC , reduce_size , thread_size ); + + kokkos_omp_in_critical_region = 0 ; + } +/* END #pragma omp critical */ + } +/* END #pragma omp parallel */ + } +} + +} // namespace Impl +} // namespace Kokkos + +//---------------------------------------------------------------------------- +//---------------------------------------------------------------------------- + +namespace Kokkos { + +//---------------------------------------------------------------------------- + +int OpenMP::is_initialized() +{ return 0 != Impl::OpenMPexec::m_pool[0]; } + +void OpenMP::initialize( unsigned thread_count , + unsigned use_numa_count , + unsigned use_cores_per_numa ) +{ + // Before any other call to OMP query the maximum number of threads + // and save the value for re-initialization unit testing. + static int omp_max_threads = omp_get_max_threads(); + + const bool is_initialized = 0 != Impl::OpenMPexec::m_pool[0] ; + + bool thread_spawn_failed = false ; + + if ( ! is_initialized ) { + + // Use hwloc thread pinning if concerned with locality. + // If spreading threads across multiple NUMA regions. + // If hyperthreading is enabled. + Impl::s_using_hwloc = hwloc::available() && ( + ( 1 < Kokkos::hwloc::get_available_numa_count() ) || + ( 1 < Kokkos::hwloc::get_available_threads_per_core() ) ); + + std::pair threads_coord[ Impl::OpenMPexec::MAX_THREAD_COUNT ]; + + // If hwloc available then use it's maximum value. + + if ( thread_count == 0 ) { + thread_count = Impl::s_using_hwloc + ? Kokkos::hwloc::get_available_numa_count() * + Kokkos::hwloc::get_available_cores_per_numa() * + Kokkos::hwloc::get_available_threads_per_core() + : omp_max_threads ; + } + + if(Impl::s_using_hwloc) + hwloc::thread_mapping( "Kokkos::OpenMP::initialize" , + false /* do not allow asynchronous */ , + thread_count , + use_numa_count , + use_cores_per_numa , + threads_coord ); + + // Spawn threads: + + omp_set_num_threads( thread_count ); + + // Verify OMP interaction: + if ( int(thread_count) != omp_get_max_threads() ) { + thread_spawn_failed = true ; + } + + // Verify spawning and bind threads: +#pragma omp parallel + { +#pragma omp critical + { + if ( int(thread_count) != omp_get_num_threads() ) { + thread_spawn_failed = true ; + } + + // Call to 'bind_this_thread' is not thread safe so place this whole block in a critical region. + // Call to 'new' may not be thread safe as well. + + // Reverse the rank for threads so that the scan operation reduces to the highest rank thread. + + const unsigned omp_rank = omp_get_thread_num(); + const unsigned thread_r = Impl::s_using_hwloc ? Kokkos::hwloc::bind_this_thread( thread_count , threads_coord ) : omp_rank ; + + Impl::OpenMPexec::m_map_rank[ omp_rank ] = thread_r ; + } +/* END #pragma omp critical */ + } +/* END #pragma omp parallel */ + + if ( ! thread_spawn_failed ) { + Impl::OpenMPexec::m_pool_topo[0] = thread_count ; + Impl::OpenMPexec::m_pool_topo[1] = Impl::s_using_hwloc ? thread_count / use_numa_count : thread_count; + Impl::OpenMPexec::m_pool_topo[2] = Impl::s_using_hwloc ? thread_count / ( use_numa_count * use_cores_per_numa ) : 1; + + Impl::OpenMPexec::resize_scratch( 1024 , 1024 ); + } + } + + if ( is_initialized || thread_spawn_failed ) { + std::string msg("Kokkos::OpenMP::initialize ERROR"); + + if ( is_initialized ) { msg.append(" : already initialized"); } + if ( thread_spawn_failed ) { msg.append(" : failed spawning threads"); } + + Kokkos::Impl::throw_runtime_exception(msg); + } +} + +//---------------------------------------------------------------------------- + +void OpenMP::finalize() +{ + Impl::OpenMPexec::verify_initialized( "OpenMP::finalize" ); + Impl::OpenMPexec::verify_is_process( "OpenMP::finalize" ); + + Impl::OpenMPexec::clear_scratch(); + + Impl::OpenMPexec::m_pool_topo[0] = 0 ; + Impl::OpenMPexec::m_pool_topo[1] = 0 ; + Impl::OpenMPexec::m_pool_topo[2] = 0 ; + + omp_set_num_threads(0); + + if ( Impl::s_using_hwloc ) { + hwloc::unbind_this_thread(); + } +} + +//---------------------------------------------------------------------------- + +void OpenMP::print_configuration( std::ostream & s , const bool detail ) +{ + Impl::OpenMPexec::verify_is_process( "OpenMP::print_configuration" ); + + s << "Kokkos::OpenMP" ; + +#if defined( KOKKOS_HAVE_OPENMP ) + s << " KOKKOS_HAVE_OPENMP" ; +#endif +#if defined( KOKKOS_HAVE_HWLOC ) + + const unsigned numa_count = Kokkos::hwloc::get_available_numa_count(); + const unsigned cores_per_numa = Kokkos::hwloc::get_available_cores_per_numa(); + const unsigned threads_per_core = Kokkos::hwloc::get_available_threads_per_core(); + + s << " hwloc[" << numa_count << "x" << cores_per_numa << "x" << threads_per_core << "]" + << " hwloc_binding_" << ( Impl::s_using_hwloc ? "enabled" : "disabled" ) + ; +#endif + + const bool is_initialized = 0 != Impl::OpenMPexec::m_pool[0] ; + + if ( is_initialized ) { + const int numa_count = Kokkos::Impl::OpenMPexec::m_pool_topo[0] / Kokkos::Impl::OpenMPexec::m_pool_topo[1] ; + const int core_per_numa = Kokkos::Impl::OpenMPexec::m_pool_topo[1] / Kokkos::Impl::OpenMPexec::m_pool_topo[2] ; + const int thread_per_core = Kokkos::Impl::OpenMPexec::m_pool_topo[2] ; + + s << " thread_pool_topology[ " << numa_count + << " x " << core_per_numa + << " x " << thread_per_core + << " ]" + << std::endl ; + + if ( detail ) { + std::vector< std::pair > coord( Kokkos::Impl::OpenMPexec::m_pool_topo[0] ); + +#pragma omp parallel + { +#pragma omp critical + { + coord[ omp_get_thread_num() ] = hwloc::get_this_thread_coordinate(); + } +/* END #pragma omp critical */ + } +/* END #pragma omp parallel */ + + for ( unsigned i = 0 ; i < coord.size() ; ++i ) { + s << " thread omp_rank[" << i << "]" + << " kokkos_rank[" << Impl::OpenMPexec::m_map_rank[ i ] << "]" + << " hwloc_coord[" << coord[i].first << "." << coord[i].second << "]" + << std::endl ; + } + } + } + else { + s << " not initialized" << std::endl ; + } +} + +} // namespace Kokkos + +#endif //KOKKOS_HAVE_OPENMP diff --git a/lib/kokkos/core/src/OpenMP/Kokkos_OpenMPexec.hpp b/lib/kokkos/core/src/OpenMP/Kokkos_OpenMPexec.hpp new file mode 100755 index 0000000000..82b27b97bc --- /dev/null +++ b/lib/kokkos/core/src/OpenMP/Kokkos_OpenMPexec.hpp @@ -0,0 +1,758 @@ +/* +//@HEADER +// ************************************************************************ +// +// Kokkos: Manycore Performance-Portable Multidimensional Arrays +// Copyright (2012) Sandia Corporation +// +// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +// the U.S. Government retains certain rights in this software. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// 1. Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// 3. Neither the name of the Corporation nor the names of the +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Questions? Contact H. Carter Edwards (hcedwar@sandia.gov) +// +// ************************************************************************ +//@HEADER +*/ + +#ifndef KOKKOS_OPENMPEXEC_HPP +#define KOKKOS_OPENMPEXEC_HPP + +#include +#include + +#include + +namespace Kokkos { +namespace Impl { + +//---------------------------------------------------------------------------- +/** \brief Data for OpenMP thread execution */ + +class OpenMPexec { +public: + + enum { MAX_THREAD_COUNT = 4096 }; + +private: + + static int m_pool_topo[ 4 ]; + static int m_map_rank[ MAX_THREAD_COUNT ]; + static OpenMPexec * m_pool[ MAX_THREAD_COUNT ]; // Indexed by: m_pool_rank_rev + + friend class Kokkos::OpenMP ; + + int const m_pool_rank ; + int const m_pool_rank_rev ; + int const m_scratch_exec_end ; + int const m_scratch_reduce_end ; + int const m_scratch_thread_end ; + + int volatile m_barrier_state ; + + OpenMPexec(); + OpenMPexec( const OpenMPexec & ); + OpenMPexec & operator = ( const OpenMPexec & ); + + static void clear_scratch(); + +public: + + // Topology of a cache coherent thread pool: + // TOTAL = NUMA x GRAIN + // pool_size( depth = 0 ) + // pool_size(0) = total number of threads + // pool_size(1) = number of threads per NUMA + // pool_size(2) = number of threads sharing finest grain memory hierarchy + + inline static + int pool_size( int depth = 0 ) { return m_pool_topo[ depth ]; } + + inline static + OpenMPexec * pool_rev( int pool_rank_rev ) { return m_pool[ pool_rank_rev ]; } + + inline int pool_rank() const { return m_pool_rank ; } + inline int pool_rank_rev() const { return m_pool_rank_rev ; } + + inline void * scratch_reduce() const { return ((char *) this) + m_scratch_exec_end ; } + inline void * scratch_thread() const { return ((char *) this) + m_scratch_reduce_end ; } + + inline + void state_wait( int state ) + { Impl::spinwait( m_barrier_state , state ); } + + inline + void state_set( int state ) { m_barrier_state = state ; } + + ~OpenMPexec() {} + + OpenMPexec( const int poolRank + , const int scratch_exec_size + , const int scratch_reduce_size + , const int scratch_thread_size ) + : m_pool_rank( poolRank ) + , m_pool_rank_rev( pool_size() - ( poolRank + 1 ) ) + , m_scratch_exec_end( scratch_exec_size ) + , m_scratch_reduce_end( m_scratch_exec_end + scratch_reduce_size ) + , m_scratch_thread_end( m_scratch_reduce_end + scratch_thread_size ) + , m_barrier_state(0) + {} + + static void finalize(); + + static void initialize( const unsigned team_count , + const unsigned threads_per_team , + const unsigned numa_count , + const unsigned cores_per_numa ); + + static void verify_is_process( const char * const ); + static void verify_initialized( const char * const ); + + static void resize_scratch( size_t reduce_size , size_t thread_size ); + + inline static + OpenMPexec * get_thread_omp() { return m_pool[ m_map_rank[ omp_get_thread_num() ] ]; } +}; + +} // namespace Impl +} // namespace Kokkos + +//---------------------------------------------------------------------------- +//---------------------------------------------------------------------------- + +namespace Kokkos { +namespace Impl { + +class OpenMPexecTeamMember { +private: + + enum { TEAM_REDUCE_SIZE = 512 }; + + /** \brief Thread states for team synchronization */ + enum { Active = 0 , Rendezvous = 1 }; + + typedef Kokkos::OpenMP execution_space ; + typedef execution_space::scratch_memory_space scratch_memory_space ; + + Impl::OpenMPexec & m_exec ; + scratch_memory_space m_team_shared ; + int m_team_shmem ; + int m_team_base_rev ; + int m_team_rank_rev ; + int m_team_rank ; + int m_team_size ; + int m_league_rank ; + int m_league_end ; + int m_league_size ; + + // Fan-in team threads, root of the fan-in which does not block returns true + inline + bool team_fan_in() const + { + for ( int n = 1 , j ; ( ( j = m_team_rank_rev + n ) < m_team_size ) && ! ( m_team_rank_rev & n ) ; n <<= 1 ) { + m_exec.pool_rev( m_team_base_rev + j )->state_wait( Active ); + } + + if ( m_team_rank_rev ) { + m_exec.state_set( Rendezvous ); + m_exec.state_wait( Rendezvous ); + } + + return 0 == m_team_rank_rev ; + } + + inline + void team_fan_out() const + { + for ( int n = 1 , j ; ( ( j = m_team_rank_rev + n ) < m_team_size ) && ! ( m_team_rank_rev & n ) ; n <<= 1 ) { + m_exec.pool_rev( m_team_base_rev + j )->state_set( Active ); + } + } + +public: + + KOKKOS_INLINE_FUNCTION + const execution_space::scratch_memory_space & team_shmem() const + { return m_team_shared ; } + + KOKKOS_INLINE_FUNCTION int league_rank() const { return m_league_rank ; } + KOKKOS_INLINE_FUNCTION int league_size() const { return m_league_size ; } + KOKKOS_INLINE_FUNCTION int team_rank() const { return m_team_rank ; } + KOKKOS_INLINE_FUNCTION int team_size() const { return m_team_size ; } + + KOKKOS_INLINE_FUNCTION void team_barrier() const +#if ! defined( KOKKOS_ACTIVE_EXECUTION_MEMORY_SPACE_HOST ) + {} +#else + { + if ( 1 < m_team_size ) { + team_fan_in(); + team_fan_out(); + } + } +#endif + + template + KOKKOS_INLINE_FUNCTION + void team_broadcast(ValueType& value, const int& thread_id) const + { +#if ! defined( KOKKOS_ACTIVE_EXECUTION_MEMORY_SPACE_HOST ) + { } +#else + // Make sure there is enough scratch space: + typedef typename if_c< sizeof(ValueType) < TEAM_REDUCE_SIZE + , ValueType , void >::type type ; + + type * const local_value = ((type*) m_exec.scratch_thread()); + if(team_rank() == thread_id) + *local_value = value; + memory_fence(); + team_barrier(); + value = *local_value; +#endif + } + +#ifdef KOKKOS_HAVE_CXX11 + template< class ValueType, class JoinOp > + KOKKOS_INLINE_FUNCTION ValueType + team_reduce( const ValueType & value + , const JoinOp & op_in ) const + #if ! defined( KOKKOS_ACTIVE_EXECUTION_MEMORY_SPACE_HOST ) + { return ValueType(); } + #else + { + typedef ValueType value_type; + const JoinLambdaAdapter op(op_in); + #endif +#else // KOKKOS_HAVE_CXX11 + template< class JoinOp > + KOKKOS_INLINE_FUNCTION typename JoinOp::value_type + team_reduce( const typename JoinOp::value_type & value + , const JoinOp & op ) const + #if ! defined( KOKKOS_ACTIVE_EXECUTION_MEMORY_SPACE_HOST ) + { return typename JoinOp::value_type(); } + #else + { + typedef typename JoinOp::value_type value_type; + #endif +#endif // KOKKOS_HAVE_CXX11 +#if defined( KOKKOS_ACTIVE_EXECUTION_MEMORY_SPACE_HOST ) + // Make sure there is enough scratch space: + typedef typename if_c< sizeof(value_type) < TEAM_REDUCE_SIZE + , value_type , void >::type type ; + + type * const local_value = ((type*) m_exec.scratch_thread()); + + // Set this thread's contribution + *local_value = value ; + + // Fence to make sure the base team member has access: + memory_fence(); + + if ( team_fan_in() ) { + // The last thread to synchronize returns true, all other threads wait for team_fan_out() + type * const team_value = ((type*) m_exec.pool_rev( m_team_base_rev )->scratch_thread()); + + // Join to the team value: + for ( int i = 1 ; i < m_team_size ; ++i ) { + op.join( *team_value , *((type*) m_exec.pool_rev( m_team_base_rev + i )->scratch_thread()) ); + } + + // The base team member may "lap" the other team members, + // copy to their local value before proceeding. + for ( int i = 1 ; i < m_team_size ; ++i ) { + *((type*) m_exec.pool_rev( m_team_base_rev + i )->scratch_thread()) = *team_value ; + } + + // Fence to make sure all team members have access + memory_fence(); + } + + team_fan_out(); + + return *((type volatile const *)local_value); + } +#endif + /** \brief Intra-team exclusive prefix sum with team_rank() ordering + * with intra-team non-deterministic ordering accumulation. + * + * The global inter-team accumulation value will, at the end of the + * league's parallel execution, be the scan's total. + * Parallel execution ordering of the league's teams is non-deterministic. + * As such the base value for each team's scan operation is similarly + * non-deterministic. + */ + template< typename ArgType > + KOKKOS_INLINE_FUNCTION ArgType team_scan( const ArgType & value , ArgType * const global_accum ) const +#if ! defined( KOKKOS_ACTIVE_EXECUTION_MEMORY_SPACE_HOST ) + { return ArgType(); } +#else + { + // Make sure there is enough scratch space: + typedef typename if_c< sizeof(ArgType) < TEAM_REDUCE_SIZE , ArgType , void >::type type ; + + volatile type * const work_value = ((type*) m_exec.scratch_thread()); + + *work_value = value ; + + memory_fence(); + + if ( team_fan_in() ) { + // The last thread to synchronize returns true, all other threads wait for team_fan_out() + // m_team_base[0] == highest ranking team member + // m_team_base[ m_team_size - 1 ] == lowest ranking team member + // + // 1) copy from lower to higher rank, initialize lowest rank to zero + // 2) prefix sum from lowest to highest rank, skipping lowest rank + + type accum = 0 ; + + if ( global_accum ) { + for ( int i = m_team_size ; i-- ; ) { + type & val = *((type*) m_exec.pool_rev( m_team_base_rev + i )->scratch_thread()); + accum += val ; + } + accum = atomic_fetch_add( global_accum , accum ); + } + + for ( int i = m_team_size ; i-- ; ) { + type & val = *((type*) m_exec.pool_rev( m_team_base_rev + i )->scratch_thread()); + const type offset = accum ; + accum += val ; + val = offset ; + } + + memory_fence(); + } + + team_fan_out(); + + return *work_value ; + } +#endif + + /** \brief Intra-team exclusive prefix sum with team_rank() ordering. + * + * The highest rank thread can compute the reduction total as + * reduction_total = dev.team_scan( value ) + value ; + */ + template< typename Type > + KOKKOS_INLINE_FUNCTION Type team_scan( const Type & value ) const + { return this-> template team_scan( value , 0 ); } + +#ifdef KOKKOS_HAVE_CXX11 + + /** \brief Inter-thread parallel for. Executes op(iType i) for each i=0..N-1. + * + * The range i=0..N-1 is mapped to all threads of the the calling thread team. + * This functionality requires C++11 support.*/ + template< typename iType, class Operation> + KOKKOS_INLINE_FUNCTION void team_par_for(const iType n, const Operation & op) const { + const int chunk = ((n+m_team_size-1)/m_team_size); + const int start = chunk*m_team_rank; + const int end = start+chunk + inline + OpenMPexecTeamMember( Impl::OpenMPexec & exec + , const TeamPolicy< Arg0 , Arg1 , Kokkos::OpenMP > & team + , const int shmem_size + ) + : m_exec( exec ) + , m_team_shared(0,0) + , m_team_shmem( shmem_size ) + , m_team_base_rev(0) + , m_team_rank_rev(0) + , m_team_rank(0) + , m_team_size( team.team_size() ) + , m_league_rank(0) + , m_league_end(0) + , m_league_size( team.league_size() ) + { + const int pool_rank_rev = m_exec.pool_rank_rev(); + const int pool_team_rank_rev = pool_rank_rev % team.team_alloc(); + const int pool_league_rank_rev = pool_rank_rev / team.team_alloc(); + const int league_iter_end = team.league_size() - pool_league_rank_rev * team.team_iter(); + + if ( pool_team_rank_rev < m_team_size && 0 < league_iter_end ) { + m_team_base_rev = team.team_alloc() * pool_league_rank_rev ; + m_team_rank_rev = pool_team_rank_rev ; + m_team_rank = m_team_size - ( m_team_rank_rev + 1 ); + m_league_end = league_iter_end ; + m_league_rank = league_iter_end > team.team_iter() ? league_iter_end - team.team_iter() : 0 ; + new( (void*) &m_team_shared ) space( ( (char*) m_exec.pool_rev(m_team_base_rev)->scratch_thread() ) + TEAM_REDUCE_SIZE , m_team_shmem ); + } + } + + bool valid() const + { return m_league_rank < m_league_end ; } + + void next() + { + if ( ++m_league_rank < m_league_end ) { + team_barrier(); + new( (void*) &m_team_shared ) space( ( (char*) m_exec.pool_rev(m_team_base_rev)->scratch_thread() ) + TEAM_REDUCE_SIZE , m_team_shmem ); + } + } + + static inline int team_reduce_size() { return TEAM_REDUCE_SIZE ; } +}; + + + +} // namespace Impl + +template< class Arg0 , class Arg1 > +class TeamPolicy< Arg0 , Arg1 , Kokkos::OpenMP > +{ +public: + + //! Tag this class as a kokkos execution policy + typedef TeamPolicy execution_policy ; + + //! Execution space of this execution policy. + typedef Kokkos::OpenMP execution_space ; + + typedef typename + Impl::if_c< ! Impl::is_same< Kokkos::OpenMP , Arg0 >::value , Arg0 , Arg1 >::type + work_tag ; + + //---------------------------------------- + + template< class FunctorType > + inline static + int team_size_max( const FunctorType & ) + { return execution_space::thread_pool_size(1); } + + template< class FunctorType > + inline static + int team_size_recommended( const FunctorType & ) + { return execution_space::thread_pool_size(2); } + + //---------------------------------------- + +private: + + int m_league_size ; + int m_team_size ; + int m_team_alloc ; + int m_team_iter ; + + inline void init( const int league_size_request + , const int team_size_request ) + { + const int pool_size = execution_space::thread_pool_size(0); + const int team_max = execution_space::thread_pool_size(1); + const int team_grain = execution_space::thread_pool_size(2); + + m_league_size = league_size_request ; + + m_team_size = team_size_request < team_max ? + team_size_request : team_max ; + + // Round team size up to a multiple of 'team_gain' + const int team_size_grain = team_grain * ( ( m_team_size + team_grain - 1 ) / team_grain ); + const int team_count = pool_size / team_size_grain ; + + // Constraint : pool_size = m_team_alloc * team_count + m_team_alloc = pool_size / team_count ; + + // Maxumum number of iterations each team will take: + m_team_iter = ( m_league_size + team_count - 1 ) / team_count ; + } + +public: + + inline int team_size() const { return m_team_size ; } + inline int league_size() const { return m_league_size ; } + + /** \brief Specify league size, request team size */ + TeamPolicy( execution_space & , int league_size_request , int team_size_request , int vector_length_request = 1) + { init( league_size_request , team_size_request ); (void) vector_length_request; } + + TeamPolicy( int league_size_request , int team_size_request , int vector_length_request = 1 ) + { init( league_size_request , team_size_request ); (void) vector_length_request; } + + inline int team_alloc() const { return m_team_alloc ; } + inline int team_iter() const { return m_team_iter ; } + + typedef Impl::OpenMPexecTeamMember member_type ; +}; + +} // namespace Kokkos + +//---------------------------------------------------------------------------- +//---------------------------------------------------------------------------- + +namespace Kokkos { + +inline +int OpenMP::thread_pool_size( int depth ) +{ + return Impl::OpenMPexec::pool_size(depth); +} + +KOKKOS_INLINE_FUNCTION +int OpenMP::thread_pool_rank() +{ +#if defined( KOKKOS_ACTIVE_EXECUTION_MEMORY_SPACE_HOST ) + return Impl::OpenMPexec::m_map_rank[ omp_get_thread_num() ]; +#else + return -1 ; +#endif +} + +} // namespace Kokkos + + +#ifdef KOKKOS_HAVE_CXX11 + +namespace Kokkos { + +template +KOKKOS_INLINE_FUNCTION +Impl::TeamThreadLoopBoundariesStruct + TeamThreadLoop(const Impl::OpenMPexecTeamMember& thread, const iType& count) { + return Impl::TeamThreadLoopBoundariesStruct(thread,count); +} + +template +KOKKOS_INLINE_FUNCTION +Impl::ThreadVectorLoopBoundariesStruct + ThreadVectorLoop(const Impl::OpenMPexecTeamMember& thread, const iType& count) { + return Impl::ThreadVectorLoopBoundariesStruct(thread,count); +} + +KOKKOS_INLINE_FUNCTION +Impl::ThreadSingleStruct PerTeam(const Impl::OpenMPexecTeamMember& thread) { + return Impl::ThreadSingleStruct(thread); +} + +KOKKOS_INLINE_FUNCTION +Impl::VectorSingleStruct PerThread(const Impl::OpenMPexecTeamMember& thread) { + return Impl::VectorSingleStruct(thread); +} +} // namespace Kokkos + +namespace Kokkos { + + /** \brief Inter-thread parallel_for. Executes lambda(iType i) for each i=0..N-1. + * + * The range i=0..N-1 is mapped to all threads of the the calling thread team. + * This functionality requires C++11 support.*/ +template +KOKKOS_INLINE_FUNCTION +void parallel_for(const Impl::TeamThreadLoopBoundariesStruct& loop_boundaries, const Lambda& lambda) { + for( iType i = loop_boundaries.start; i < loop_boundaries.end; i+=loop_boundaries.increment) + lambda(i); +} + +/** \brief Inter-thread vector parallel_reduce. Executes lambda(iType i, ValueType & val) for each i=0..N-1. + * + * The range i=0..N-1 is mapped to all threads of the the calling thread team and a summation of + * val is performed and put into result. This functionality requires C++11 support.*/ +template< typename iType, class Lambda, typename ValueType > +KOKKOS_INLINE_FUNCTION +void parallel_reduce(const Impl::TeamThreadLoopBoundariesStruct& loop_boundaries, + const Lambda & lambda, ValueType& result) { + + result = ValueType(); + + for( iType i = loop_boundaries.start; i < loop_boundaries.end; i+=loop_boundaries.increment) { + ValueType tmp = ValueType(); + lambda(i,tmp); + result+=tmp; + } + + result = loop_boundaries.thread.team_reduce(result,Impl::JoinAdd()); +} + +/** \brief Intra-thread vector parallel_reduce. Executes lambda(iType i, ValueType & val) for each i=0..N-1. + * + * The range i=0..N-1 is mapped to all vector lanes of the the calling thread and a reduction of + * val is performed using JoinType(ValueType& val, const ValueType& update) and put into init_result. + * The input value of init_result is used as initializer for temporary variables of ValueType. Therefore + * the input value should be the neutral element with respect to the join operation (e.g. '0 for +-' or + * '1 for *'). This functionality requires C++11 support.*/ +template< typename iType, class Lambda, typename ValueType, class JoinType > +KOKKOS_INLINE_FUNCTION +void parallel_reduce(const Impl::TeamThreadLoopBoundariesStruct& loop_boundaries, + const Lambda & lambda, const JoinType& join, ValueType& init_result) { + + ValueType result = init_result; + + for( iType i = loop_boundaries.start; i < loop_boundaries.end; i+=loop_boundaries.increment) { + ValueType tmp = ValueType(); + lambda(i,tmp); + join(result,tmp); + } + + init_result = loop_boundaries.thread.team_reduce(result,join); +} + +} //namespace Kokkos + + +namespace Kokkos { +/** \brief Intra-thread vector parallel_for. Executes lambda(iType i) for each i=0..N-1. + * + * The range i=0..N-1 is mapped to all vector lanes of the the calling thread. + * This functionality requires C++11 support.*/ +template +KOKKOS_INLINE_FUNCTION +void parallel_for(const Impl::ThreadVectorLoopBoundariesStruct& + loop_boundaries, const Lambda& lambda) { + #ifdef KOKKOS_HAVE_PRAGMA_IVDEP + #pragma ivdep + #endif + for( iType i = loop_boundaries.start; i < loop_boundaries.end; i+=loop_boundaries.increment) + lambda(i); +} + +/** \brief Intra-thread vector parallel_reduce. Executes lambda(iType i, ValueType & val) for each i=0..N-1. + * + * The range i=0..N-1 is mapped to all vector lanes of the the calling thread and a summation of + * val is performed and put into result. This functionality requires C++11 support.*/ +template< typename iType, class Lambda, typename ValueType > +KOKKOS_INLINE_FUNCTION +void parallel_reduce(const Impl::ThreadVectorLoopBoundariesStruct& + loop_boundaries, const Lambda & lambda, ValueType& result) { + result = ValueType(); +#ifdef KOKKOS_HAVE_PRAGMA_IVDEP +#pragma ivdep +#endif + for( iType i = loop_boundaries.start; i < loop_boundaries.end; i+=loop_boundaries.increment) { + ValueType tmp = ValueType(); + lambda(i,tmp); + result+=tmp; + } +} + +/** \brief Intra-thread vector parallel_reduce. Executes lambda(iType i, ValueType & val) for each i=0..N-1. + * + * The range i=0..N-1 is mapped to all vector lanes of the the calling thread and a reduction of + * val is performed using JoinType(ValueType& val, const ValueType& update) and put into init_result. + * The input value of init_result is used as initializer for temporary variables of ValueType. Therefore + * the input value should be the neutral element with respect to the join operation (e.g. '0 for +-' or + * '1 for *'). This functionality requires C++11 support.*/ +template< typename iType, class Lambda, typename ValueType, class JoinType > +KOKKOS_INLINE_FUNCTION +void parallel_reduce(const Impl::ThreadVectorLoopBoundariesStruct& + loop_boundaries, const Lambda & lambda, const JoinType& join, ValueType& init_result) { + + ValueType result = init_result; +#ifdef KOKKOS_HAVE_PRAGMA_IVDEP +#pragma ivdep +#endif + for( iType i = loop_boundaries.start; i < loop_boundaries.end; i+=loop_boundaries.increment) { + ValueType tmp = ValueType(); + lambda(i,tmp); + join(result,tmp); + } + init_result = result; +} + +/** \brief Intra-thread vector parallel exclusive prefix sum. Executes lambda(iType i, ValueType & val, bool final) + * for each i=0..N-1. + * + * The range i=0..N-1 is mapped to all vector lanes in the thread and a scan operation is performed. + * Depending on the target execution space the operator might be called twice: once with final=false + * and once with final=true. When final==true val contains the prefix sum value. The contribution of this + * "i" needs to be added to val no matter whether final==true or not. In a serial execution + * (i.e. team_size==1) the operator is only called once with final==true. Scan_val will be set + * to the final sum value over all vector lanes. + * This functionality requires C++11 support.*/ +template< typename iType, class FunctorType > +KOKKOS_INLINE_FUNCTION +void parallel_scan(const Impl::ThreadVectorLoopBoundariesStruct& + loop_boundaries, const FunctorType & lambda) { + + typedef Kokkos::Impl::FunctorValueTraits< FunctorType , void > ValueTraits ; + typedef typename ValueTraits::value_type value_type ; + + value_type scan_val = value_type(); + +#ifdef KOKKOS_HAVE_PRAGMA_IVDEP +#pragma ivdep +#endif + for( iType i = loop_boundaries.start; i < loop_boundaries.end; i+=loop_boundaries.increment) { + lambda(i,scan_val,true); + } +} + +} // namespace Kokkos + +namespace Kokkos { + +template +KOKKOS_INLINE_FUNCTION +void single(const Impl::VectorSingleStruct& single_struct, const FunctorType& lambda) { + lambda(); +} + +template +KOKKOS_INLINE_FUNCTION +void single(const Impl::ThreadSingleStruct& single_struct, const FunctorType& lambda) { + if(single_struct.team_member.team_rank()==0) lambda(); +} + +template +KOKKOS_INLINE_FUNCTION +void single(const Impl::VectorSingleStruct& single_struct, const FunctorType& lambda, ValueType& val) { + lambda(val); +} + +template +KOKKOS_INLINE_FUNCTION +void single(const Impl::ThreadSingleStruct& single_struct, const FunctorType& lambda, ValueType& val) { + if(single_struct.team_member.team_rank()==0) { + lambda(val); + } + single_struct.team_member.team_broadcast(val,0); +} +} + +#endif // KOKKOS_HAVE_CXX11 + +#endif /* #ifndef KOKKOS_OPENMPEXEC_HPP */ + diff --git a/lib/kokkos/core/src/Qthread/Kokkos_QthreadExec.cpp b/lib/kokkos/core/src/Qthread/Kokkos_QthreadExec.cpp new file mode 100755 index 0000000000..ca76784a5c --- /dev/null +++ b/lib/kokkos/core/src/Qthread/Kokkos_QthreadExec.cpp @@ -0,0 +1,380 @@ +/* +//@HEADER +// ************************************************************************ +// +// Kokkos: Manycore Performance-Portable Multidimensional Arrays +// Copyright (2012) Sandia Corporation +// +// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +// the U.S. Government retains certain rights in this software. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// 1. Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// 3. Neither the name of the Corporation nor the names of the +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Questions? Contact H. Carter Edwards (hcedwar@sandia.gov) +// +// ************************************************************************ +//@HEADER +*/ + +#include + +#if defined( KOKKOS_HAVE_QTHREAD ) + +#include +#include +#include +#include +#include +#include +#include +#include + +#define QTHREAD_LOCAL_PRIORITY + +#include + +//---------------------------------------------------------------------------- + +namespace Kokkos { +namespace Impl { +namespace { + +enum { MAXIMUM_QTHREAD_WORKERS = 1024 }; + +/** s_exec is indexed by the reverse rank of the workers + * for faster fan-in / fan-out lookups + * [ n - 1 , n - 2 , ... , 0 ] + */ +QthreadExec * s_exec[ MAXIMUM_QTHREAD_WORKERS ]; + +int s_number_shepherds = 0 ; +int s_number_workers_per_shepherd = 0 ; +int s_number_workers = 0 ; + +inline +QthreadExec ** worker_exec() +{ + return s_exec + s_number_workers - ( qthread_shep() * s_number_workers_per_shepherd + qthread_worker_local(NULL) + 1 ); +} + +const int s_base_size = QthreadExec::align_alloc( sizeof(QthreadExec) ); + +int s_worker_reduce_end = 0 ; /* End of worker reduction memory */ +int s_worker_shared_end = 0 ; /* Total of worker scratch memory */ +int s_worker_shared_begin = 0 ; /* Beginning of worker shared memory */ + +QthreadExecFunctionPointer s_active_function = 0 ; +const void * s_active_function_arg = 0 ; + +} /* namespace */ +} /* namespace Impl */ +} /* namespace Kokkos */ + +//---------------------------------------------------------------------------- + +namespace Kokkos { + +void Qthread::initialize( int thread_count ) +{ + // Environment variable: QTHREAD_NUM_SHEPHERDS + // Environment variable: QTHREAD_NUM_WORKERS_PER_SHEP + // Environment variable: QTHREAD_HWPAR + + { + char buffer[256]; + snprintf(buffer,sizeof(buffer),"QTHREAD_HWPAR=%d",thread_count); + putenv(buffer); + } + + const bool ok_init = ( QTHREAD_SUCCESS == qthread_initialize() ) && + ( thread_count == qthread_num_shepherds() * qthread_num_workers_local(NO_SHEPHERD) ) && + ( thread_count == qthread_num_workers() ); + + bool ok_symmetry = true ; + + if ( ok_init ) { + Impl::s_number_shepherds = qthread_num_shepherds(); + Impl::s_number_workers_per_shepherd = qthread_num_workers_local(NO_SHEPHERD); + Impl::s_number_workers = Impl::s_number_shepherds * Impl::s_number_workers_per_shepherd ; + + for ( int i = 0 ; ok_symmetry && i < Impl::s_number_shepherds ; ++i ) { + ok_symmetry = ( Impl::s_number_workers_per_shepherd == qthread_num_workers_local(i) ); + } + } + + if ( ! ok_init || ! ok_symmetry ) { + std::ostringstream msg ; + + msg << "Kokkos::Qthread::initialize(" << thread_count << ") FAILED" ; + msg << " : qthread_num_shepherds = " << qthread_num_shepherds(); + msg << " : qthread_num_workers_per_shepherd = " << qthread_num_workers_local(NO_SHEPHERD); + msg << " : qthread_num_workers = " << qthread_num_workers(); + + if ( ! ok_symmetry ) { + msg << " : qthread_num_workers_local = {" ; + for ( int i = 0 ; i < Impl::s_number_shepherds ; ++i ) { + msg << " " << qthread_num_workers_local(i) ; + } + msg << " }" ; + } + + Impl::s_number_workers = 0 ; + Impl::s_number_shepherds = 0 ; + Impl::s_number_workers_per_shepherd = 0 ; + + if ( ok_init ) { qthread_finalize(); } + + Kokkos::Impl::throw_runtime_exception( msg.str() ); + } + + Impl::QthreadExec::resize_worker_scratch( 256 , 256 ); +} + +void Qthread::finalize() +{ + Impl::QthreadExec::clear_workers(); + + if ( Impl::s_number_workers ) { + qthread_finalize(); + } + + Impl::s_number_workers = 0 ; + Impl::s_number_shepherds = 0 ; + Impl::s_number_workers_per_shepherd = 0 ; +} + +void Qthread::print_configuration( std::ostream & s , const bool detail ) +{ + s << "Kokkos::Qthread {" + << " num_shepherds(" << Impl::s_number_shepherds << ")" + << " num_workers_per_shepherd(" << Impl::s_number_workers_per_shepherd << ")" + << " }" << std::endl ; +} + +Qthread & Qthread::instance( int ) +{ + static Qthread q ; + return q ; +} + +void Qthread::fence() +{ +} + +int Qthread::shepherd_size() const { return Impl::s_number_shepherds ; } +int Qthread::shepherd_worker_size() const { return Impl::s_number_workers_per_shepherd ; } + +} /* namespace Kokkos */ + +//---------------------------------------------------------------------------- + +namespace Kokkos { +namespace Impl { +namespace { + +aligned_t driver_exec_all( void * arg ) +{ + (*s_active_function)( ** worker_exec() , s_active_function_arg ); + + return 0 ; +} + +aligned_t driver_resize_worker_scratch( void * arg ) +{ + static volatile int lock_begin = 0 ; + static volatile int lock_end = 0 ; + + QthreadExec ** const exec = worker_exec(); + + //---------------------------------------- + // Serialize allocation for thread safety + + while ( ! atomic_compare_exchange_strong( & lock_begin , 0 , 1 ) ); // Spin wait to claim lock + + const bool ok = 0 == *exec ; + + if ( ok ) { *exec = (QthreadExec *) malloc( s_base_size + s_worker_shared_end ); } + + lock_begin = 0 ; // release lock + + if ( ok ) { new( *exec ) QthreadExec(); } + + //---------------------------------------- + // Wait for all calls to complete to insure that each worker has executed. + + if ( s_number_workers == 1 + atomic_fetch_add( & lock_end , 1 ) ) { lock_end = 0 ; } + + while ( lock_end ); + + //---------------------------------------- + + return 0 ; +} + +void verify_is_process( const char * const label , bool not_active = false ) +{ + const bool not_process = 0 != qthread_shep() || 0 != qthread_worker_local(NULL); + const bool is_active = not_active && ( s_active_function || s_active_function_arg ); + + if ( not_process || is_active ) { + std::string msg( label ); + msg.append( " : FAILED" ); + if ( not_process ) msg.append(" : not called by main process"); + if ( is_active ) msg.append(" : parallel execution in progress"); + Kokkos::Impl::throw_runtime_exception( msg ); + } +} + +} + +QthreadExec::QthreadExec() +{ + const int shepherd_rank = qthread_shep(); + const int shepherd_worker_rank = qthread_worker_local(NULL); + const int worker_rank = shepherd_rank * s_number_workers_per_shepherd + shepherd_worker_rank ; + + m_worker_base = s_exec ; + m_shepherd_base = s_exec + s_number_workers_per_shepherd * ( ( s_number_shepherds - ( shepherd_rank + 1 ) ) ); + m_scratch_alloc = ( (unsigned char *) this ) + s_base_size ; + m_reduce_end = s_worker_reduce_end ; + m_shepherd_rank = shepherd_rank ; + m_shepherd_size = s_number_shepherds ; + m_shepherd_worker_rank = shepherd_worker_rank ; + m_shepherd_worker_size = s_number_workers_per_shepherd ; + m_worker_rank = worker_rank ; + m_worker_size = s_number_workers ; + m_worker_state = QthreadExec::Active ; +} + +void QthreadExec::clear_workers() +{ + for ( int iwork = 0 ; iwork < s_number_workers ; ++iwork ) { + free( s_exec[iwork] ); + s_exec[iwork] = 0 ; + } +} + +void QthreadExec::shared_reset( Qthread::scratch_memory_space & space ) +{ + new( & space ) + Qthread::scratch_memory_space( + ((unsigned char *) (**m_shepherd_base).m_scratch_alloc ) + s_worker_shared_begin , + s_worker_shared_end - s_worker_shared_begin + ); +} + +void QthreadExec::resize_worker_scratch( const int reduce_size , const int shared_size ) +{ + const int exec_all_reduce_alloc = align_alloc( reduce_size ); + const int shepherd_scan_alloc = align_alloc( 8 ); + const int shepherd_shared_end = exec_all_reduce_alloc + shepherd_scan_alloc + align_alloc( shared_size ); + + if ( s_worker_reduce_end < exec_all_reduce_alloc || + s_worker_shared_end < shepherd_shared_end ) { + + // Clear current worker memory before allocating new worker memory + clear_workers(); + + // Increase the buffers to an aligned allocation + s_worker_reduce_end = exec_all_reduce_alloc ; + s_worker_shared_begin = exec_all_reduce_alloc + shepherd_scan_alloc ; + s_worker_shared_end = shepherd_shared_end ; + + // Need to query which shepherd this main 'process' is running... + + // Have each worker resize its memory for proper first-touch + for ( int jshep = 0 ; jshep < s_number_shepherds ; ++jshep ) { + for ( int i = jshep ? 0 : 1 ; i < s_number_workers_per_shepherd ; ++i ) { + + // Unit tests hang with this call: + // + // qthread_fork_to_local_priority( driver_resize_workers , NULL , NULL , jshep ); + // + + qthread_fork_to( driver_resize_worker_scratch , NULL , NULL , jshep ); + }} + + driver_resize_worker_scratch( NULL ); + + // Verify all workers allocated + + bool ok = true ; + for ( int iwork = 0 ; ok && iwork < s_number_workers ; ++iwork ) { ok = 0 != s_exec[iwork] ; } + + if ( ! ok ) { + std::ostringstream msg ; + msg << "Kokkos::Impl::QthreadExec::resize : FAILED for workers {" ; + for ( int iwork = 0 ; iwork < s_number_workers ; ++iwork ) { + if ( 0 == s_exec[iwork] ) { msg << " " << ( s_number_workers - ( iwork + 1 ) ); } + } + msg << " }" ; + Kokkos::Impl::throw_runtime_exception( msg.str() ); + } + } +} + +void QthreadExec::exec_all( Qthread & , QthreadExecFunctionPointer func , const void * arg ) +{ + verify_is_process("QthreadExec::exec_all(...)",true); + + s_active_function = func ; + s_active_function_arg = arg ; + + // Need to query which shepherd this main 'process' is running... + + const int main_shep = qthread_shep(); + + for ( int jshep = 0 , iwork = 0 ; jshep < s_number_shepherds ; ++jshep ) { + for ( int i = jshep != main_shep ? 0 : 1 ; i < s_number_workers_per_shepherd ; ++i , ++iwork ) { + + // Unit tests hang with this call: + // + // qthread_fork_to_local_priority( driver_exec_all , NULL , NULL , jshep ); + // + + qthread_fork_to( driver_exec_all , NULL , NULL , jshep ); + }} + + driver_exec_all( NULL ); + + s_active_function = 0 ; + s_active_function_arg = 0 ; +} + +void * QthreadExec::exec_all_reduce_result() +{ + return s_exec[0]->m_scratch_alloc ; +} + +} /* namespace Impl */ +} /* namespace Kokkos */ + +//---------------------------------------------------------------------------- + +#endif /* #if defined( KOKKOS_HAVE_QTHREAD ) */ + diff --git a/lib/kokkos/core/src/Qthread/Kokkos_QthreadExec.hpp b/lib/kokkos/core/src/Qthread/Kokkos_QthreadExec.hpp new file mode 100755 index 0000000000..5ed544c130 --- /dev/null +++ b/lib/kokkos/core/src/Qthread/Kokkos_QthreadExec.hpp @@ -0,0 +1,580 @@ +/* +//@HEADER +// ************************************************************************ +// +// Kokkos: Manycore Performance-Portable Multidimensional Arrays +// Copyright (2012) Sandia Corporation +// +// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +// the U.S. Government retains certain rights in this software. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// 1. Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// 3. Neither the name of the Corporation nor the names of the +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Questions? Contact H. Carter Edwards (hcedwar@sandia.gov) +// +// ************************************************************************ +//@HEADER +*/ + +#ifndef KOKKOS_QTHREADEXEC_HPP +#define KOKKOS_QTHREADEXEC_HPP + +#include + +//---------------------------------------------------------------------------- + +namespace Kokkos { +namespace Impl { + +//---------------------------------------------------------------------------- + +class QthreadExec ; + +typedef void (*QthreadExecFunctionPointer)( QthreadExec & , const void * ); + +class QthreadExec { +private: + + enum { Inactive = 0 , Active = 1 }; + + const QthreadExec * const * m_worker_base ; + const QthreadExec * const * m_shepherd_base ; + + void * m_scratch_alloc ; ///< Scratch memory [ reduce , team , shared ] + int m_reduce_end ; ///< End of scratch reduction memory + + int m_shepherd_rank ; + int m_shepherd_size ; + + int m_shepherd_worker_rank ; + int m_shepherd_worker_size ; + + /* + * m_worker_rank = m_shepherd_rank * m_shepherd_worker_size + m_shepherd_worker_rank + * m_worker_size = m_shepherd_size * m_shepherd_worker_size + */ + int m_worker_rank ; + int m_worker_size ; + + int mutable volatile m_worker_state ; + + + friend class Kokkos::Qthread ; + + ~QthreadExec(); + QthreadExec( const QthreadExec & ); + QthreadExec & operator = ( const QthreadExec & ); + +public: + + QthreadExec(); + + /** Execute the input function on all available Qthread workers */ + static void exec_all( Qthread & , QthreadExecFunctionPointer , const void * ); + + //---------------------------------------- + /** Barrier across all workers participating in the 'exec_all' */ + void exec_all_barrier() const + { + const int rev_rank = m_worker_size - ( m_worker_rank + 1 ); + + int n , j ; + + for ( n = 1 ; ( ! ( rev_rank & n ) ) && ( ( j = rev_rank + n ) < m_worker_size ) ; n <<= 1 ) { + Impl::spinwait( m_worker_base[j]->m_worker_state , QthreadExec::Active ); + } + + if ( rev_rank ) { + m_worker_state = QthreadExec::Inactive ; + Impl::spinwait( m_worker_state , QthreadExec::Inactive ); + } + + for ( n = 1 ; ( ! ( rev_rank & n ) ) && ( ( j = rev_rank + n ) < m_worker_size ) ; n <<= 1 ) { + m_worker_base[j]->m_worker_state = QthreadExec::Active ; + } + } + + /** Barrier across workers within the shepherd with rank < team_rank */ + void shepherd_barrier( const int team_size ) const + { + if ( m_shepherd_worker_rank < team_size ) { + + const int rev_rank = team_size - ( m_shepherd_worker_rank + 1 ); + + int n , j ; + + for ( n = 1 ; ( ! ( rev_rank & n ) ) && ( ( j = rev_rank + n ) < team_size ) ; n <<= 1 ) { + Impl::spinwait( m_shepherd_base[j]->m_worker_state , QthreadExec::Active ); + } + + if ( rev_rank ) { + m_worker_state = QthreadExec::Inactive ; + Impl::spinwait( m_worker_state , QthreadExec::Inactive ); + } + + for ( n = 1 ; ( ! ( rev_rank & n ) ) && ( ( j = rev_rank + n ) < team_size ) ; n <<= 1 ) { + m_shepherd_base[j]->m_worker_state = QthreadExec::Active ; + } + } + } + + //---------------------------------------- + /** Reduce across all workers participating in the 'exec_all' */ + template< class FunctorType , class ArgTag > + inline + void exec_all_reduce( const FunctorType & func ) const + { + typedef Kokkos::Impl::FunctorValueJoin< FunctorType , ArgTag > ValueJoin ; + typedef Kokkos::Impl::FunctorValueOps< FunctorType , ArgTag > ValueOps ; + + const int rev_rank = m_worker_size - ( m_worker_rank + 1 ); + + int n , j ; + + for ( n = 1 ; ( ! ( rev_rank & n ) ) && ( ( j = rev_rank + n ) < m_worker_size ) ; n <<= 1 ) { + const QthreadExec & fan = *m_worker_base[j]; + + Impl::spinwait( fan.m_worker_state , QthreadExec::Active ); + + ValueJoin::join( func , m_scratch_alloc , fan.m_scratch_alloc ); + } + + if ( rev_rank ) { + m_worker_state = QthreadExec::Inactive ; + Impl::spinwait( m_worker_state , QthreadExec::Inactive ); + } + + for ( n = 1 ; ( ! ( rev_rank & n ) ) && ( ( j = rev_rank + n ) < m_worker_size ) ; n <<= 1 ) { + m_worker_base[j]->m_worker_state = QthreadExec::Active ; + } + } + + //---------------------------------------- + /** Scall across all workers participating in the 'exec_all' */ + template< class FunctorType , class ArgTag > + inline + void exec_all_scan( const FunctorType & func ) const + { + typedef Kokkos::Impl::FunctorValueInit< FunctorType , ArgTag > ValueInit ; + typedef Kokkos::Impl::FunctorValueJoin< FunctorType , ArgTag > ValueJoin ; + typedef Kokkos::Impl::FunctorValueOps< FunctorType , ArgTag > ValueOps ; + + const int rev_rank = m_worker_size - ( m_worker_rank + 1 ); + + int n , j ; + + for ( n = 1 ; ( ! ( rev_rank & n ) ) && ( ( j = rev_rank + n ) < m_worker_size ) ; n <<= 1 ) { + Impl::spinwait( m_worker_base[j]->m_worker_state , QthreadExec::Active ); + } + + if ( rev_rank ) { + m_worker_state = QthreadExec::Inactive ; + Impl::spinwait( m_worker_state , QthreadExec::Inactive ); + } + else { + // Root thread scans across values before releasing threads + // Worker data is in reverse order, so m_worker_base[0] is the + // highest ranking thread. + + // Copy from lower ranking to higher ranking worker. + for ( int i = 1 ; i < n ; ++i ) { + ValueOps::copy( func , m_worker_base[i-1]->m_scratch_alloc + , m_worker_base[i]->m_scratch_alloc ); + } + + ValueInit::init( func , m_worker_base[n-1]->m_scratch_alloc ); + + // Join from lower ranking to higher ranking worker. + // Value at m_worker_base[n-1] is zero so skip adding it to m_worker_base[n-2]. + for ( int i = n - 1 ; --i ; ) { + ValueJoin::join( func , m_worker_base[i-1]->m_scratch_alloc , m_worker_base[i]->m_scratch_alloc ); + } + } + + for ( n = 1 ; ( ! ( rev_rank & n ) ) && ( ( j = rev_rank + n ) < m_worker_size ) ; n <<= 1 ) { + m_worker_base[j]->m_worker_state = QthreadExec::Active ; + } + } + + //---------------------------------------- + + template< class Type> + inline + volatile Type * shepherd_team_scratch_value() const + { return (volatile Type*)(((unsigned char *) m_scratch_alloc) + m_reduce_end); } + + template< class Type > + inline + Type shepherd_reduce( const int team_size , const Type & value ) const + { + *shepherd_team_scratch_value() = value ; + + memory_fence(); + + const int rev_rank = team_size - ( m_shepherd_worker_rank + 1 ); + + int n , j ; + + for ( n = 1 ; ( ! ( rev_rank & n ) ) && ( ( j = rev_rank + n ) < team_size ) ; n <<= 1 ) { + Impl::spinwait( m_shepherd_base[j]->m_worker_state , QthreadExec::Active ); + } + + if ( rev_rank ) { + m_worker_state = QthreadExec::Inactive ; + Impl::spinwait( m_worker_state , QthreadExec::Inactive ); + } + else { + Type & accum = * m_shepherd_base[0]->shepherd_team_scratch_value(); + for ( int i = 1 ; i < n ; ++i ) { + accum += * m_shepherd_base[i]->shepherd_team_scratch_value(); + } + for ( int i = 1 ; i < n ; ++i ) { + * m_shepherd_base[i]->shepherd_team_scratch_value() = accum ; + } + + memory_fence(); + } + + for ( n = 1 ; ( ! ( rev_rank & n ) ) && ( ( j = rev_rank + n ) < team_size ) ; n <<= 1 ) { + m_shepherd_base[j]->m_worker_state = QthreadExec::Active ; + } + + return *shepherd_team_scratch_value(); + } + + template< class JoinOp > + inline + typename JoinOp::value_type + shepherd_reduce( const int team_size + , const typename JoinOp::value_type & value + , const JoinOp & op ) const + { + typedef typename JoinOp::value_type Type ; + + *shepherd_team_scratch_value() = value ; + + memory_fence(); + + const int rev_rank = team_size - ( m_shepherd_worker_rank + 1 ); + + int n , j ; + + for ( n = 1 ; ( ! ( rev_rank & n ) ) && ( ( j = rev_rank + n ) < team_size ) ; n <<= 1 ) { + Impl::spinwait( m_shepherd_base[j]->m_worker_state , QthreadExec::Active ); + } + + if ( rev_rank ) { + m_worker_state = QthreadExec::Inactive ; + Impl::spinwait( m_worker_state , QthreadExec::Inactive ); + } + else { + volatile Type & accum = * m_shepherd_base[0]->shepherd_team_scratch_value(); + for ( int i = 1 ; i < n ; ++i ) { + op.join( accum , * m_shepherd_base[i]->shepherd_team_scratch_value() ); + } + for ( int i = 1 ; i < n ; ++i ) { + * m_shepherd_base[i]->shepherd_team_scratch_value() = accum ; + } + + memory_fence(); + } + + for ( n = 1 ; ( ! ( rev_rank & n ) ) && ( ( j = rev_rank + n ) < team_size ) ; n <<= 1 ) { + m_shepherd_base[j]->m_worker_state = QthreadExec::Active ; + } + + return *shepherd_team_scratch_value(); + } + + template< class Type > + inline + Type shepherd_scan( const int team_size + , const Type & value + , Type * const global_value = 0 ) const + { + *shepherd_team_scratch_value() = value ; + + memory_fence(); + + const int rev_rank = team_size - ( m_shepherd_worker_rank + 1 ); + + int n , j ; + + for ( n = 1 ; ( ! ( rev_rank & n ) ) && ( ( j = rev_rank + n ) < team_size ) ; n <<= 1 ) { + Impl::spinwait( m_shepherd_base[j]->m_worker_state , QthreadExec::Active ); + } + + if ( rev_rank ) { + m_worker_state = QthreadExec::Inactive ; + Impl::spinwait( m_worker_state , QthreadExec::Inactive ); + } + else { + // Root thread scans across values before releasing threads + // Worker data is in reverse order, so m_shepherd_base[0] is the + // highest ranking thread. + + // Copy from lower ranking to higher ranking worker. + + Type accum = * m_shepherd_base[0]->shepherd_team_scratch_value(); + for ( int i = 1 ; i < n ; ++i ) { + const Type tmp = * m_shepherd_base[i]->shepherd_team_scratch_value(); + accum += tmp ; + * m_shepherd_base[i-1]->shepherd_team_scratch_value() = tmp ; + } + + * m_shepherd_base[n-1]->shepherd_team_scratch_value() = + global_value ? atomic_fetch_add( global_value , accum ) : 0 ; + + // Join from lower ranking to higher ranking worker. + for ( int i = n ; --i ; ) { + * m_shepherd_base[i-1]->shepherd_team_scratch_value() += * m_shepherd_base[i]->shepherd_team_scratch_value(); + } + + memory_fence(); + } + + for ( n = 1 ; ( ! ( rev_rank & n ) ) && ( ( j = rev_rank + n ) < team_size ) ; n <<= 1 ) { + m_shepherd_base[j]->m_worker_state = QthreadExec::Active ; + } + + return *shepherd_team_scratch_value(); + } + + //---------------------------------------- + + static inline + int align_alloc( int size ) + { + enum { ALLOC_GRAIN = 1 << 6 /* power of two, 64bytes */}; + enum { ALLOC_GRAIN_MASK = ALLOC_GRAIN - 1 }; + return ( size + ALLOC_GRAIN_MASK ) & ~ALLOC_GRAIN_MASK ; + } + + void shared_reset( Qthread::scratch_memory_space & ); + + void * exec_all_reduce_value() const { return m_scratch_alloc ; } + + static void * exec_all_reduce_result(); + + static void resize_worker_scratch( const int reduce_size , const int shared_size ); + static void clear_workers(); + + //---------------------------------------- + + inline int worker_rank() const { return m_worker_rank ; } + inline int worker_size() const { return m_worker_size ; } + inline int shepherd_worker_rank() const { return m_shepherd_worker_rank ; } + inline int shepherd_worker_size() const { return m_shepherd_worker_size ; } + inline int shepherd_rank() const { return m_shepherd_rank ; } + inline int shepherd_size() const { return m_shepherd_size ; } +}; + +} /* namespace Impl */ +} /* namespace Kokkos */ + +//---------------------------------------------------------------------------- + +namespace Kokkos { +namespace Impl { + +class QthreadTeamPolicyMember { +private: + + typedef Kokkos::Qthread execution_space ; + typedef execution_space::scratch_memory_space scratch_memory_space ; + + + Impl::QthreadExec & m_exec ; + scratch_memory_space m_team_shared ; + const int m_team_size ; + const int m_team_rank ; + const int m_league_size ; + const int m_league_end ; + int m_league_rank ; + +public: + + KOKKOS_INLINE_FUNCTION + const scratch_memory_space & team_shmem() const { return m_team_shared ; } + + KOKKOS_INLINE_FUNCTION int league_rank() const { return m_league_rank ; } + KOKKOS_INLINE_FUNCTION int league_size() const { return m_league_size ; } + KOKKOS_INLINE_FUNCTION int team_rank() const { return m_team_rank ; } + KOKKOS_INLINE_FUNCTION int team_size() const { return m_team_size ; } + + KOKKOS_INLINE_FUNCTION void team_barrier() const +#if ! defined( KOKKOS_ACTIVE_EXECUTION_MEMORY_SPACE_HOST ) + {} +#else + { m_exec.shepherd_barrier( m_team_size ); } +#endif + + template< typename Type > + KOKKOS_INLINE_FUNCTION Type team_reduce( const Type & value ) const +#if ! defined( KOKKOS_ACTIVE_EXECUTION_MEMORY_SPACE_HOST ) + { return Type(); } +#else + { return m_exec.template shepherd_reduce( m_team_size , value ); } +#endif + + template< typename JoinOp > + KOKKOS_INLINE_FUNCTION typename JoinOp::value_type + team_reduce( const typename JoinOp::value_type & value + , const JoinOp & op ) const +#if ! defined( KOKKOS_ACTIVE_EXECUTION_MEMORY_SPACE_HOST ) + { return typename JoinOp::value_type(); } +#else + { return m_exec.template shepherd_reduce( m_team_size , value , op ); } +#endif + + /** \brief Intra-team exclusive prefix sum with team_rank() ordering. + * + * The highest rank thread can compute the reduction total as + * reduction_total = dev.team_scan( value ) + value ; + */ + template< typename Type > + KOKKOS_INLINE_FUNCTION Type team_scan( const Type & value ) const +#if ! defined( KOKKOS_ACTIVE_EXECUTION_MEMORY_SPACE_HOST ) + { return Type(); } +#else + { return m_exec.template shepherd_scan( m_team_size , value ); } +#endif + + /** \brief Intra-team exclusive prefix sum with team_rank() ordering + * with intra-team non-deterministic ordering accumulation. + * + * The global inter-team accumulation value will, at the end of the + * league's parallel execution, be the scan's total. + * Parallel execution ordering of the league's teams is non-deterministic. + * As such the base value for each team's scan operation is similarly + * non-deterministic. + */ + template< typename Type > + KOKKOS_INLINE_FUNCTION Type team_scan( const Type & value , Type * const global_accum ) const +#if ! defined( KOKKOS_ACTIVE_EXECUTION_MEMORY_SPACE_HOST ) + { return Type(); } +#else + { return m_exec.template shepherd_scan( m_team_size , value , global_accum ); } +#endif + + //---------------------------------------- + // Private for the driver ( for ( member_type i(exec,team); i ; i.next_team() ) { ... } + + // Initialize + template< class Arg0 , class Arg1 > + QthreadTeamPolicyMember( Impl::QthreadExec & exec , const TeamPolicy & team ) + : m_exec( exec ) + , m_team_shared(0,0) + , m_team_size( team.m_team_size ) + , m_team_rank( exec.shepherd_worker_rank() ) + , m_league_size( team.m_league_size ) + , m_league_end( team.m_league_size - team.m_shepherd_iter * ( exec.shepherd_size() - ( exec.shepherd_rank() + 1 ) ) ) + , m_league_rank( m_league_end > team.m_shepherd_iter ? m_league_end - team.m_shepherd_iter : 0 ) + { + m_exec.shared_reset( m_team_shared ); + } + + // Continue + operator bool () const { return m_league_rank < m_league_end ; } + + // iterate + void next_team() { ++m_league_rank ; m_exec.shared_reset( m_team_shared ); } +}; + +} // namespace Impl + +template< class Arg0 , class Arg1 > +class TeamPolicy< Arg0 , Arg1 , Kokkos::Qthread > +{ +private: + + const int m_league_size ; + const int m_team_size ; + const int m_shepherd_iter ; + +public: + + //! Tag this class as a kokkos execution policy + typedef TeamPolicy execution_policy ; + typedef Qthread execution_space ; + + typedef typename + Impl::if_c< ! Impl::is_same< Kokkos::Qthread , Arg0 >::value , Arg0 , Arg1 >::type + work_tag ; + + //---------------------------------------- + + template< class FunctorType > + inline static + int team_size_max( const FunctorType & ) + { return Qthread::instance().shepherd_worker_size(); } + + template< class FunctorType > + static int team_size_recommended( const FunctorType & f ) + { return team_size_max( f ); } + + //---------------------------------------- + + inline int team_size() const { return m_team_size ; } + inline int league_size() const { return m_league_size ; } + + // One active team per shepherd + TeamPolicy( Kokkos::Qthread & q + , const int league_size + , const int team_size + ) + : m_league_size( league_size ) + , m_team_size( team_size < q.shepherd_worker_size() + ? team_size : q.shepherd_worker_size() ) + , m_shepherd_iter( ( league_size + q.shepherd_size() - 1 ) / q.shepherd_size() ) + { + } + + // One active team per shepherd + TeamPolicy( const int league_size + , const int team_size + ) + : m_league_size( league_size ) + , m_team_size( team_size < Qthread::instance().shepherd_worker_size() + ? team_size : Qthread::instance().shepherd_worker_size() ) + , m_shepherd_iter( ( league_size + Qthread::instance().shepherd_size() - 1 ) / Qthread::instance().shepherd_size() ) + { + } + + typedef Impl::QthreadTeamPolicyMember member_type ; + + friend class Impl::QthreadTeamPolicyMember ; +}; + +} /* namespace Kokkos */ + +//---------------------------------------------------------------------------- +//---------------------------------------------------------------------------- + +#endif /* #define KOKKOS_QTHREADEXEC_HPP */ + diff --git a/lib/kokkos/core/src/Qthread/Kokkos_Qthread_Parallel.hpp b/lib/kokkos/core/src/Qthread/Kokkos_Qthread_Parallel.hpp new file mode 100755 index 0000000000..ab89c05198 --- /dev/null +++ b/lib/kokkos/core/src/Qthread/Kokkos_Qthread_Parallel.hpp @@ -0,0 +1,418 @@ +/* +//@HEADER +// ************************************************************************ +// +// Kokkos: Manycore Performance-Portable Multidimensional Arrays +// Copyright (2012) Sandia Corporation +// +// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +// the U.S. Government retains certain rights in this software. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// 1. Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// 3. Neither the name of the Corporation nor the names of the +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Questions? Contact H. Carter Edwards (hcedwar@sandia.gov) +// +// ************************************************************************ +//@HEADER +*/ + +#ifndef KOKKOS_QTHREAD_PARALLEL_HPP +#define KOKKOS_QTHREAD_PARALLEL_HPP + +#include + +#include + +#include +#include + +#include + +//---------------------------------------------------------------------------- + +namespace Kokkos { +namespace Impl { + +//---------------------------------------------------------------------------- + +template< class FunctorType , class Arg0 , class Arg1 , class Arg2 > +class ParallelFor< FunctorType , Kokkos::RangePolicy< Arg0 , Arg1 , Arg2 , Kokkos::Qthread > > +{ +private: + + typedef Kokkos::RangePolicy< Arg0 , Arg1 , Arg2 , Kokkos::Qthread > Policy ; + + const FunctorType m_func ; + const Policy m_policy ; + + template< class PType > + KOKKOS_FORCEINLINE_FUNCTION static + void driver( typename Impl::enable_if< + ( Impl::is_same< typename PType::work_tag , void >::value ) + , const FunctorType & >::type functor + , const PType & range ) + { + const typename PType::member_type e = range.end(); + for ( typename PType::member_type i = range.begin() ; i < e ; ++i ) { + functor( i ); + } + } + + template< class PType > + KOKKOS_FORCEINLINE_FUNCTION static + void driver( typename Impl::enable_if< + ( ! Impl::is_same< typename PType::work_tag , void >::value ) + , const FunctorType & >::type functor + , const PType & range ) + { + const typename PType::member_type e = range.end(); + for ( typename PType::member_type i = range.begin() ; i < e ; ++i ) { + functor( typename PType::work_tag() , i ); + } + } + + // Function is called once by every concurrent thread. + static void execute( QthreadExec & exec , const void * arg ) + { + const ParallelFor & self = * ((const ParallelFor *) arg ); + + driver( self.m_func , typename Policy::WorkRange( self.m_policy , exec.worker_rank() , exec.worker_size() ) ); + + // All threads wait for completion. + exec.exec_all_barrier(); + } + +public: + + ParallelFor( const FunctorType & functor + , const Policy & policy + ) + : m_func( functor ) + , m_policy( policy ) + { + Impl::QthreadExec::exec_all( Qthread::instance() , & ParallelFor::execute , this ); + } +}; + +//---------------------------------------------------------------------------- + +template< class FunctorType , class Arg0 , class Arg1 , class Arg2 > +class ParallelReduce< FunctorType , Kokkos::RangePolicy< Arg0 , Arg1 , Arg2 , Kokkos::Qthread > > +{ +private: + + typedef Kokkos::RangePolicy< Arg0 , Arg1 , Arg2 , Kokkos::Qthread > Policy ; + typedef Kokkos::Impl::FunctorValueTraits< FunctorType , typename Policy::work_tag > ValueTraits ; + typedef Kokkos::Impl::FunctorValueInit< FunctorType , typename Policy::work_tag > ValueInit ; + + typedef typename ValueTraits::pointer_type pointer_type ; + typedef typename ValueTraits::reference_type reference_type ; + + const FunctorType m_func ; + const Policy m_policy ; + + template< class PType > + KOKKOS_FORCEINLINE_FUNCTION static + void driver( typename Impl::enable_if< + ( Impl::is_same< typename PType::work_tag , void >::value ) + , const FunctorType & >::type functor + , reference_type update + , const PType & range ) + { + const typename PType::member_type e = range.end(); + for ( typename PType::member_type i = range.begin() ; i < e ; ++i ) { + functor( i , update ); + } + } + + template< class PType > + KOKKOS_FORCEINLINE_FUNCTION static + void driver( typename Impl::enable_if< + ( ! Impl::is_same< typename PType::work_tag , void >::value ) + , const FunctorType & >::type functor + , reference_type update + , const PType & range ) + { + const typename PType::member_type e = range.end(); + for ( typename PType::member_type i = range.begin() ; i < e ; ++i ) { + functor( typename PType::work_tag() , i , update ); + } + } + + static void execute( QthreadExec & exec , const void * arg ) + { + const ParallelReduce & self = * ((const ParallelReduce *) arg ); + + driver( self.m_func + , ValueInit::init( self.m_func , exec.exec_all_reduce_value() ) + , typename Policy::WorkRange( self.m_policy , exec.worker_rank() , exec.worker_size() ) + ); + + exec.template exec_all_reduce( self.m_func ); + } + +public: + + template< class HostViewType > + ParallelReduce( const FunctorType & functor + , const Policy & policy + , const HostViewType & result_view ) + : m_func( functor ) + , m_policy( policy ) + { + QthreadExec::resize_worker_scratch( ValueTraits::value_size( m_func ) , 0 ); + + Impl::QthreadExec::exec_all( Qthread::instance() , & ParallelReduce::execute , this ); + + const pointer_type data = (pointer_type) QthreadExec::exec_all_reduce_result(); + + Kokkos::Impl::FunctorFinal< FunctorType , typename Policy::work_tag >::final( m_func , data ); + + if ( result_view.ptr_on_device() ) { + const unsigned n = ValueTraits::value_count( m_func ); + for ( unsigned i = 0 ; i < n ; ++i ) { result_view.ptr_on_device()[i] = data[i]; } + } + } +}; + +//---------------------------------------------------------------------------- + +template< class FunctorType , class Arg0 , class Arg1 > +class ParallelFor< FunctorType , TeamPolicy< Arg0 , Arg1 , Kokkos::Qthread > > +{ +private: + + typedef TeamPolicy< Arg0 , Arg1 , Kokkos::Qthread > Policy ; + + const FunctorType m_func ; + const Policy m_team ; + + template< class TagType > + KOKKOS_FORCEINLINE_FUNCTION + void driver( typename Impl::enable_if< Impl::is_same< TagType , void >::value , + const typename Policy::member_type & >::type member ) const + { m_func( member ); } + + template< class TagType > + KOKKOS_FORCEINLINE_FUNCTION + void driver( typename Impl::enable_if< ! Impl::is_same< TagType , void >::value , + const typename Policy::member_type & >::type member ) const + { m_func( TagType() , member ); } + + static void execute( QthreadExec & exec , const void * arg ) + { + const ParallelFor & self = * ((const ParallelFor *) arg ); + + typename Policy::member_type member( exec , self.m_team ); + + while ( member ) { + self.ParallelFor::template driver< typename Policy::work_tag >( member ); + member.team_barrier(); + member.next_team(); + } + + exec.exec_all_barrier(); + } + +public: + + ParallelFor( const FunctorType & functor , + const Policy & policy ) + : m_func( functor ) + , m_team( policy ) + { + QthreadExec::resize_worker_scratch + ( /* reduction memory */ 0 + , /* team shared memory */ FunctorTeamShmemSize< FunctorType >::value( functor , policy.team_size() ) ); + + Impl::QthreadExec::exec_all( Qthread::instance() , & ParallelFor::execute , this ); + } +}; + +//---------------------------------------------------------------------------- + +template< class FunctorType , class Arg0 , class Arg1 > +class ParallelReduce< FunctorType , TeamPolicy< Arg0 , Arg1 , Kokkos::Qthread > > +{ +private: + + typedef TeamPolicy< Arg0 , Arg1 , Kokkos::Qthread > Policy ; + + typedef Kokkos::Impl::FunctorValueTraits< FunctorType , typename Policy::work_tag > ValueTraits ; + typedef Kokkos::Impl::FunctorValueInit< FunctorType , typename Policy::work_tag > ValueInit ; + + typedef typename ValueTraits::pointer_type pointer_type ; + typedef typename ValueTraits::reference_type reference_type ; + + const FunctorType m_func ; + const Policy m_team ; + + template< class TagType > + KOKKOS_FORCEINLINE_FUNCTION + void driver( typename Impl::enable_if< Impl::is_same< TagType , void >::value , + const typename Policy::member_type & >::type member + , reference_type update ) const + { m_func( member , update ); } + + template< class TagType > + KOKKOS_FORCEINLINE_FUNCTION + void driver( typename Impl::enable_if< ! Impl::is_same< TagType , void >::value , + const typename Policy::member_type & >::type member + , reference_type update ) const + { m_func( TagType() , member , update ); } + + static void execute( QthreadExec & exec , const void * arg ) + { + const ParallelReduce & self = * ((const ParallelReduce *) arg ); + + // Initialize thread-local value + reference_type update = ValueInit::init( self.m_func , exec.exec_all_reduce_value() ); + + typename Policy::member_type member( exec , self.m_team ); + + while ( member ) { + self.ParallelReduce::template driver< typename Policy::work_tag >( member , update ); + member.team_barrier(); + member.next_team(); + } + + exec.template exec_all_reduce< FunctorType , typename Policy::work_tag >( self.m_func ); + } + +public: + + template< class ViewType > + ParallelReduce( const FunctorType & functor , + const Policy & policy , + const ViewType & result ) + : m_func( functor ) + , m_team( policy ) + { + QthreadExec::resize_worker_scratch + ( /* reduction memory */ ValueTraits::value_size( functor ) + , /* team shared memory */ FunctorTeamShmemSize< FunctorType >::value( functor , policy.team_size() ) ); + + Impl::QthreadExec::exec_all( Qthread::instance() , & ParallelReduce::execute , this ); + + const pointer_type data = (pointer_type) QthreadExec::exec_all_reduce_result(); + + Kokkos::Impl::FunctorFinal< FunctorType , typename Policy::work_tag >::final( m_func , data ); + + const unsigned n = ValueTraits::value_count( m_func ); + for ( unsigned i = 0 ; i < n ; ++i ) { result.ptr_on_device()[i] = data[i]; } + } +}; + +//---------------------------------------------------------------------------- +//---------------------------------------------------------------------------- + +template< class FunctorType , class Arg0 , class Arg1 , class Arg2 > +class ParallelScan< FunctorType , Kokkos::RangePolicy< Arg0 , Arg1 , Arg2 , Kokkos::Qthread > > +{ +private: + + typedef Kokkos::RangePolicy< Arg0 , Arg1 , Arg2 , Kokkos::Qthread > Policy ; + typedef Kokkos::Impl::FunctorValueTraits< FunctorType , typename Policy::work_tag > ValueTraits ; + typedef Kokkos::Impl::FunctorValueInit< FunctorType , typename Policy::work_tag > ValueInit ; + + typedef typename ValueTraits::pointer_type pointer_type ; + typedef typename ValueTraits::reference_type reference_type ; + + const FunctorType m_func ; + const Policy m_policy ; + + template< class PType > + KOKKOS_FORCEINLINE_FUNCTION static + void driver( typename Impl::enable_if< + ( Impl::is_same< typename PType::work_tag , void >::value ) + , const FunctorType & >::type functor + , reference_type update + , const bool final + , const PType & range ) + { + const typename PType::member_type e = range.end(); + for ( typename PType::member_type i = range.begin() ; i < e ; ++i ) { + functor( i , update , final ); + } + } + + template< class PType > + KOKKOS_FORCEINLINE_FUNCTION static + void driver( typename Impl::enable_if< + ( ! Impl::is_same< typename PType::work_tag , void >::value ) + , const FunctorType & >::type functor + , reference_type update + , const bool final + , const PType & range ) + { + const typename PType::member_type e = range.end(); + for ( typename PType::member_type i = range.begin() ; i < e ; ++i ) { + functor( typename PType::work_tag() , i , update , final ); + } + } + + static void execute( QthreadExec & exec , const void * arg ) + { + const ParallelScan & self = * ((const ParallelScan *) arg ); + + const typename Policy::WorkRange range( self.m_policy , exec.worker_rank() , exec.worker_size() ); + + // Initialize thread-local value + reference_type update = ValueInit::init( self.m_func , exec.exec_all_reduce_value() ); + + driver( self.m_func , update , false , range ); + + exec.template exec_all_scan< FunctorType , typename Policy::work_tag >( self.m_func ); + + driver( self.m_func , update , true , range ); + + exec.exec_all_barrier(); + } + +public: + + ParallelScan( const FunctorType & functor + , const Policy & policy + ) + : m_func( functor ) + , m_policy( policy ) + { + QthreadExec::resize_worker_scratch( ValueTraits::value_size( m_func ) , 0 ); + + Impl::QthreadExec::exec_all( Qthread::instance() , & ParallelScan::execute , this ); + } +}; + +} // namespace Impl +} // namespace Kokkos + +//---------------------------------------------------------------------------- +//---------------------------------------------------------------------------- + +#endif /* #define KOKKOS_QTHREAD_PARALLEL_HPP */ + diff --git a/lib/kokkos/core/src/Qthread/Kokkos_Qthread_TaskPolicy.cpp b/lib/kokkos/core/src/Qthread/Kokkos_Qthread_TaskPolicy.cpp new file mode 100755 index 0000000000..b830079afa --- /dev/null +++ b/lib/kokkos/core/src/Qthread/Kokkos_Qthread_TaskPolicy.cpp @@ -0,0 +1,299 @@ +/* +//@HEADER +// ************************************************************************ +// +// Kokkos: Manycore Performance-Portable Multidimensional Arrays +// Copyright (2012) Sandia Corporation +// +// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +// the U.S. Government retains certain rights in this software. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// 1. Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// 3. Neither the name of the Corporation nor the names of the +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Questions? Contact H. Carter Edwards (hcedwar@sandia.gov) +// +// ************************************************************************ +//@HEADER +*/ + +// Experimental unified task-data parallel manycore LDRD + +#include + +#if defined( KOKKOS_HAVE_QTHREAD ) + +#include + +#include +#include +#include +#include +#include + +#include +#include + +//---------------------------------------------------------------------------- + +namespace Kokkos { +namespace Impl { + +typedef TaskMember< Kokkos::Qthread , void , void > Task ; + +namespace { + +inline +unsigned padded_sizeof_derived( unsigned sizeof_derived ) +{ + return sizeof_derived + + ( sizeof_derived % sizeof(Task*) ? sizeof(Task*) - sizeof_derived % sizeof(Task*) : 0 ); +} + +} // namespace + +void Task::deallocate( void * ptr ) +{ + // Counting on 'free' thread safety so lock/unlock not required. + // However, isolate calls here to mitigate future need to introduce lock/unlock. + + // lock + + free( ptr ); + + // unlock +} + +void * Task::allocate( const unsigned arg_sizeof_derived + , const unsigned arg_dependence_capacity ) +{ + // Counting on 'malloc' thread safety so lock/unlock not required. + // However, isolate calls here to mitigate future need to introduce lock/unlock. + + // lock + + void * const ptr = malloc( padded_sizeof_derived( arg_sizeof_derived ) + arg_dependence_capacity * sizeof(Task*) ); + + // unlock + + return ptr ; +} + +Task::~TaskMember() +{ + +} + + +Task::TaskMember( const function_verify_type arg_verify + , const function_dealloc_type arg_dealloc + , const function_apply_type arg_apply + , const unsigned arg_sizeof_derived + , const unsigned arg_dependence_capacity + ) + : m_dealloc( arg_dealloc ) + , m_verify( arg_verify ) + , m_apply( arg_apply ) + , m_dep( (Task **)( ((unsigned char *) this) + padded_sizeof_derived( arg_sizeof_derived ) ) ) + , m_dep_capacity( arg_dependence_capacity ) + , m_dep_size( 0 ) + , m_ref_count( 0 ) + , m_state( Kokkos::TASK_STATE_CONSTRUCTING ) + , m_qfeb(0) +{ + qthread_empty( & m_qfeb ); // Set to full when complete + for ( unsigned i = 0 ; i < arg_dependence_capacity ; ++i ) m_dep[i] = 0 ; +} + +Task::TaskMember( const function_dealloc_type arg_dealloc + , const function_apply_type arg_apply + , const unsigned arg_sizeof_derived + , const unsigned arg_dependence_capacity + ) + : m_dealloc( arg_dealloc ) + , m_verify( & Task::verify_type ) + , m_apply( arg_apply ) + , m_dep( (Task **)( ((unsigned char *) this) + padded_sizeof_derived( arg_sizeof_derived ) ) ) + , m_dep_capacity( arg_dependence_capacity ) + , m_dep_size( 0 ) + , m_ref_count( 0 ) + , m_state( Kokkos::TASK_STATE_CONSTRUCTING ) + , m_qfeb(0) +{ + qthread_empty( & m_qfeb ); // Set to full when complete + for ( unsigned i = 0 ; i < arg_dependence_capacity ; ++i ) m_dep[i] = 0 ; +} + +//---------------------------------------------------------------------------- + +void Task::throw_error_add_dependence() const +{ + std::cerr << "TaskMember< Qthread >::add_dependence ERROR" + << " state(" << m_state << ")" + << " dep_size(" << m_dep_size << ")" + << std::endl ; + throw std::runtime_error("TaskMember< Qthread >::add_dependence ERROR"); +} + +void Task::throw_error_verify_type() +{ + throw std::runtime_error("TaskMember< Qthread >::verify_type ERROR"); +} + +//---------------------------------------------------------------------------- + +#if defined( KOKKOS_ACTIVE_EXECUTION_MEMORY_SPACE_HOST ) +void Task::assign( Task ** const lhs , Task * rhs , const bool no_throw ) +{ + static const char msg_error_header[] = "Kokkos::Impl::TaskManager::assign ERROR" ; + static const char msg_error_count[] = ": negative reference count" ; + static const char msg_error_complete[] = ": destroy task that is not complete" ; + static const char msg_error_dependences[] = ": destroy task that has dependences" ; + static const char msg_error_exception[] = ": caught internal exception" ; + + const char * msg_error = 0 ; + + try { + + if ( *lhs ) { + + const int count = Kokkos::atomic_fetch_add( & (**lhs).m_ref_count , -1 ); + + if ( 1 == count ) { + + // Reference count at zero, delete it + + // Should only be deallocating a completed task + if ( (**lhs).m_state == Kokkos::TASK_STATE_COMPLETE ) { + + // A completed task should not have dependences... + for ( int i = 0 ; i < (**lhs).m_dep_size && 0 == msg_error ; ++i ) { + if ( (**lhs).m_dep[i] ) msg_error = msg_error_dependences ; + } + } + else { + msg_error = msg_error_complete ; + } + + if ( 0 == msg_error ) { + // Get deletion function and apply it + const Task::function_dealloc_type d = (**lhs).m_dealloc ; + + (*d)( *lhs ); + } + } + else if ( count <= 0 ) { + msg_error = msg_error_count ; + } + } + + if ( 0 == msg_error && rhs ) { Kokkos::atomic_fetch_add( & (*rhs).m_ref_count , 1 ); } + + *lhs = rhs ; + } + catch( ... ) { + if ( 0 == msg_error ) msg_error = msg_error_exception ; + } + + if ( 0 != msg_error ) { + if ( no_throw ) { + std::cerr << msg_error_header << msg_error << std::endl ; + std::cerr.flush(); + } + else { + std::string msg(msg_error_header); + msg.append(msg_error); + throw std::runtime_error( msg ); + } + } +} +#endif + + +//---------------------------------------------------------------------------- + +aligned_t Task::qthread_func( void * arg ) +{ + Task * const task = reinterpret_cast< Task * >(arg); + + task->m_state = Kokkos::TASK_STATE_EXECUTING ; + + (*task->m_apply)( task ); + + if ( task->m_state == Kokkos::TASK_STATE_EXECUTING ) { + // Task did not respawn, is complete + task->m_state = Kokkos::TASK_STATE_COMPLETE ; + + // Release dependences before allowing dependent tasks to run. + // Otherwise their is a thread race condition for removing dependences. + for ( int i = 0 ; i < task->m_dep_size ; ++i ) { + assign( & task->m_dep[i] , 0 ); + } + + // Set qthread FEB to full so that dependent tasks are allowed to execute + qthread_fill( & task->m_qfeb ); + } + + return 0 ; +} + +void Task::schedule() +{ + // Is waiting for execution + + // spawn in qthread. must malloc the precondition array and give to qthread. + // qthread will eventually free this allocation so memory will not be leaked. + + // concern with thread safety of malloc, does this need to be guarded? + aligned_t ** qprecon = (aligned_t **) malloc( ( m_dep_size + 1 ) * sizeof(aligned_t *) ); + + qprecon[0] = reinterpret_cast( uintptr_t(m_dep_size) ); + + for ( int i = 0 ; i < m_dep_size ; ++i ) { + qprecon[i+1] = & m_dep[i]->m_qfeb ; // Qthread precondition flag + } + + m_state = Kokkos::TASK_STATE_WAITING ; + + qthread_spawn( & Task::qthread_func , this , 0 , NULL + , m_dep_size , qprecon + , NO_SHEPHERD , QTHREAD_SPAWN_SIMPLE ); +} + +void Task::wait( const Future< void, Kokkos::Qthread> & f ) +{ + if ( f.m_task ) { + aligned_t tmp ; + qthread_readFF( & tmp , & f.m_task->m_qfeb ); + } +} + +} // namespace Impl +} // namespace Kokkos + +#endif /* #if defined( KOKKOS_HAVE_QTHREAD ) */ + diff --git a/lib/kokkos/core/src/Qthread/Kokkos_Qthread_TaskPolicy.hpp b/lib/kokkos/core/src/Qthread/Kokkos_Qthread_TaskPolicy.hpp new file mode 100755 index 0000000000..3764f10b36 --- /dev/null +++ b/lib/kokkos/core/src/Qthread/Kokkos_Qthread_TaskPolicy.hpp @@ -0,0 +1,736 @@ +/* +//@HEADER +// ************************************************************************ +// +// Kokkos: Manycore Performance-Portable Multidimensional Arrays +// Copyright (2012) Sandia Corporation +// +// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +// the U.S. Government retains certain rights in this software. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// 1. Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// 3. Neither the name of the Corporation nor the names of the +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Questions? Contact H. Carter Edwards (hcedwar@sandia.gov) +// +// ************************************************************************ +//@HEADER +*/ + +// Experimental unified task-data parallel manycore LDRD + +#ifndef KOKKOS_QTHREAD_TASKPOLICY_HPP +#define KOKKOS_QTHREAD_TASKPOLICY_HPP + +#include +#include +#include + +#include + +#include +#include +#include + +#include + +//---------------------------------------------------------------------------- + +namespace Kokkos { +namespace Impl { + +template<> +class TaskMember< Kokkos::Qthread , void , void > +{ +public: + + typedef void (* function_apply_type) ( TaskMember * ); + typedef void (* function_dealloc_type)( TaskMember * ); + typedef TaskMember * (* function_verify_type) ( TaskMember * ); + +private: + + const function_dealloc_type m_dealloc ; ///< Deallocation + const function_verify_type m_verify ; ///< Result type verification + const function_apply_type m_apply ; ///< Apply function + TaskMember ** const m_dep ; ///< Dependences + const int m_dep_capacity ; ///< Capacity of dependences + int m_dep_size ; ///< Actual count of dependences + int m_ref_count ; ///< Reference count + int m_state ; ///< State of the task + aligned_t m_qfeb ; ///< Qthread full/empty bit + + TaskMember() /* = delete */ ; + TaskMember( const TaskMember & ) /* = delete */ ; + TaskMember & operator = ( const TaskMember & ) /* = delete */ ; + + static aligned_t qthread_func( void * arg ); + + static void * allocate( const unsigned arg_sizeof_derived , const unsigned arg_dependence_capacity ); + static void deallocate( void * ); + + void throw_error_add_dependence() const ; + static void throw_error_verify_type(); + + template < class DerivedTaskType > + static + void deallocate( TaskMember * t ) + { + DerivedTaskType * ptr = static_cast< DerivedTaskType * >(t); + ptr->~DerivedTaskType(); + deallocate( (void *) ptr ); + } + +protected : + + ~TaskMember(); + + // Used by TaskMember< Qthread , ResultType , void > + TaskMember( const function_verify_type arg_verify + , const function_dealloc_type arg_dealloc + , const function_apply_type arg_apply + , const unsigned arg_sizeof_derived + , const unsigned arg_dependence_capacity + ); + + // Used for TaskMember< Qthread , void , void > + TaskMember( const function_dealloc_type arg_dealloc + , const function_apply_type arg_apply + , const unsigned arg_sizeof_derived + , const unsigned arg_dependence_capacity + ); + +public: + + template< typename ResultType > + KOKKOS_FUNCTION static + TaskMember * verify_type( TaskMember * t ) + { + enum { check_type = ! Impl::is_same< ResultType , void >::value }; + + if ( check_type && t != 0 ) { + + // Verify that t->m_verify is this function + const function_verify_type self = & TaskMember::template verify_type< ResultType > ; + + if ( t->m_verify != self ) { + t = 0 ; +#if defined( KOKKOS_ACTIVE_EXECUTION_MEMORY_SPACE_HOST ) + throw_error_verify_type(); +#endif + } + } + return t ; + } + + //---------------------------------------- + /* Inheritence Requirements on task types: + * typedef FunctorType::value_type value_type ; + * class DerivedTaskType + * : public TaskMember< Qthread , value_type , FunctorType > + * { ... }; + * class TaskMember< Qthread , value_type , FunctorType > + * : public TaskMember< Qthread , value_type , void > + * , public Functor + * { ... }; + * If value_type != void + * class TaskMember< Qthread , value_type , void > + * : public TaskMember< Qthread , void , void > + * + * Allocate space for DerivedTaskType followed by TaskMember*[ dependence_capacity ] + * + */ + + /** \brief Allocate and construct a single-thread task */ + template< class DerivedTaskType > + static + TaskMember * create( const typename DerivedTaskType::functor_type & arg_functor + , const unsigned arg_dependence_capacity ) + { + typedef typename DerivedTaskType::functor_type functor_type ; + typedef typename functor_type::value_type value_type ; + + DerivedTaskType * const task = + new( allocate( sizeof(DerivedTaskType) , arg_dependence_capacity ) ) + DerivedTaskType( & TaskMember::template deallocate< DerivedTaskType > + , & TaskMember::template apply_single< functor_type , value_type > + , sizeof(DerivedTaskType) + , arg_dependence_capacity + , arg_functor ); + + return static_cast< TaskMember * >( task ); + } + + /** \brief Allocate and construct a data parallel task */ + template< class DerivedTaskType > + static + TaskMember * create( const typename DerivedTaskType::policy_type & arg_policy + , const typename DerivedTaskType::functor_type & arg_functor + , const unsigned arg_dependence_capacity ) + { + DerivedTaskType * const task = + new( allocate( sizeof(DerivedTaskType) , arg_dependence_capacity ) ) + DerivedTaskType( & TaskMember::template deallocate< DerivedTaskType > + , sizeof(DerivedTaskType) + , arg_dependence_capacity + , arg_policy + , arg_functor + ); + + return static_cast< TaskMember * >( task ); + } + + void schedule(); + static void wait( const Future< void , Kokkos::Qthread > & ); + + //---------------------------------------- + + typedef FutureValueTypeIsVoidError get_result_type ; + + KOKKOS_INLINE_FUNCTION + get_result_type get() const { return get_result_type() ; } + + KOKKOS_INLINE_FUNCTION + Kokkos::TaskState get_state() const { return Kokkos::TaskState( m_state ); } + + //---------------------------------------- + +#if defined( KOKKOS_ACTIVE_EXECUTION_MEMORY_SPACE_HOST ) + static + void assign( TaskMember ** const lhs , TaskMember * const rhs , const bool no_throw = false ); +#else + KOKKOS_INLINE_FUNCTION static + void assign( TaskMember ** const lhs , TaskMember * const rhs , const bool no_throw = false ) {} +#endif + + KOKKOS_INLINE_FUNCTION + TaskMember * get_dependence( int i ) const + { return ( Kokkos::TASK_STATE_EXECUTING == m_state && 0 <= i && i < m_dep_size ) ? m_dep[i] : (TaskMember*) 0 ; } + + KOKKOS_INLINE_FUNCTION + int get_dependence() const + { return m_dep_size ; } + + KOKKOS_INLINE_FUNCTION + void clear_dependence() + { + for ( int i = 0 ; i < m_dep_size ; ++i ) assign( m_dep + i , 0 ); + m_dep_size = 0 ; + } + + KOKKOS_INLINE_FUNCTION + void add_dependence( TaskMember * before ) + { + if ( ( Kokkos::TASK_STATE_CONSTRUCTING == m_state || + Kokkos::TASK_STATE_EXECUTING == m_state ) && + m_dep_size < m_dep_capacity ) { + assign( m_dep + m_dep_size , before ); + ++m_dep_size ; + } + else { + throw_error_add_dependence(); + } + } + + //---------------------------------------- + + template< class FunctorType , class ResultType > + KOKKOS_INLINE_FUNCTION static + void apply_single( typename Impl::enable_if< ! Impl::is_same< ResultType , void >::value , TaskMember * >::type t ) + { + typedef TaskMember< Kokkos::Qthread , ResultType , FunctorType > derived_type ; + + // TaskMember< Kokkos::Qthread , ResultType , FunctorType > + // : public TaskMember< Kokkos::Qthread , ResultType , void > + // , public FunctorType + // { ... }; + + derived_type & m = * static_cast< derived_type * >( t ); + + Impl::FunctorApply< FunctorType , void , ResultType & >::apply( (FunctorType &) m , & m.m_result ); + } + + template< class FunctorType , class ResultType > + KOKKOS_INLINE_FUNCTION static + void apply_single( typename Impl::enable_if< Impl::is_same< ResultType , void >::value , TaskMember * >::type t ) + { + typedef TaskMember< Kokkos::Qthread , ResultType , FunctorType > derived_type ; + + // TaskMember< Kokkos::Qthread , ResultType , FunctorType > + // : public TaskMember< Kokkos::Qthread , ResultType , void > + // , public FunctorType + // { ... }; + + derived_type & m = * static_cast< derived_type * >( t ); + + Impl::FunctorApply< FunctorType , void , void >::apply( (FunctorType &) m ); + } +}; + +//---------------------------------------------------------------------------- +/** \brief Base class for tasks with a result value in the Qthread execution space. + * + * The FunctorType must be void because this class is accessed by the + * Future class for the task and result value. + * + * Must be derived from TaskMember 'root class' so the Future class + * can correctly static_cast from the 'root class' to this class. + */ +template < class ResultType > +class TaskMember< Kokkos::Qthread , ResultType , void > + : public TaskMember< Kokkos::Qthread , void , void > +{ +public: + + ResultType m_result ; + + typedef const ResultType & get_result_type ; + + KOKKOS_INLINE_FUNCTION + get_result_type get() const { return m_result ; } + +protected: + + typedef TaskMember< Kokkos::Qthread , void , void > task_root_type ; + typedef task_root_type::function_dealloc_type function_dealloc_type ; + typedef task_root_type::function_apply_type function_apply_type ; + + inline + TaskMember( const function_dealloc_type arg_dealloc + , const function_apply_type arg_apply + , const unsigned arg_sizeof_derived + , const unsigned arg_dependence_capacity + ) + : task_root_type( & task_root_type::template verify_type< ResultType > + , arg_dealloc + , arg_apply + , arg_sizeof_derived + , arg_dependence_capacity ) + , m_result() + {} + +}; + +template< class ResultType , class FunctorType > +class TaskMember< Kokkos::Qthread , ResultType , FunctorType > + : public TaskMember< Kokkos::Qthread , ResultType , void > + , public FunctorType +{ +public: + + typedef FunctorType functor_type ; + + typedef TaskMember< Kokkos::Qthread , void , void > task_root_type ; + typedef TaskMember< Kokkos::Qthread , ResultType , void > task_base_type ; + typedef task_root_type::function_dealloc_type function_dealloc_type ; + typedef task_root_type::function_apply_type function_apply_type ; + + inline + TaskMember( const function_dealloc_type arg_dealloc + , const function_apply_type arg_apply + , const unsigned arg_sizeof_derived + , const unsigned arg_dependence_capacity + , const functor_type & arg_functor + ) + : task_base_type( arg_dealloc , arg_apply , arg_sizeof_derived , arg_dependence_capacity ) + , functor_type( arg_functor ) + {} +}; + +} /* namespace Impl */ +} /* namespace Kokkos */ + +//---------------------------------------------------------------------------- +//---------------------------------------------------------------------------- + +namespace Kokkos { +namespace Impl { + +/** \brief ForEach task in the Qthread execution space + * + * Derived from TaskMember< Kokkos::Qthread , ResultType , FunctorType > + * so that Functor can be cast to task root type without knowing policy. + */ +template< class Arg0 , class Arg1 , class Arg2 , class ResultType , class FunctorType > +class TaskForEach< Kokkos::RangePolicy< Arg0 , Arg1 , Arg2 , Kokkos::Qthread > + , ResultType + , FunctorType > + : TaskMember< Kokkos::Qthread , ResultType , FunctorType > +{ +public: + + typedef FunctorType functor_type ; + typedef RangePolicy< Arg0 , Arg1 , Arg2 , Kokkos::Qthread > policy_type ; + +private: + + friend class Kokkos::TaskPolicy< Kokkos::Qthread > ; + friend class Kokkos::Impl::TaskMember< Kokkos::Qthread , void , void > ; + + typedef TaskMember< Kokkos::Qthread , void , void > task_root_type ; + typedef TaskMember< Kokkos::Qthread , ResultType , FunctorType > task_base_type ; + typedef task_root_type::function_dealloc_type function_dealloc_type ; + + policy_type m_policy ; + + template< class Tag > + inline + typename Impl::enable_if< Impl::is_same::value >::type + apply_policy() const + { + const typename policy_type::member_type e = m_policy.end(); + for ( typename policy_type::member_type i = m_policy.begin() ; i < e ; ++i ) { + functor_type::operator()(i); + } + } + + template< class Tag > + inline + typename Impl::enable_if< ! Impl::is_same::value >::type + apply_policy() const + { + const Tag tag ; + const typename policy_type::member_type e = m_policy.end(); + for ( typename policy_type::member_type i = m_policy.begin() ; i < e ; ++i ) { + functor_type::operator()(tag,i); + } + } + + static + void apply_parallel( task_root_type * t ) + { + static_cast(t)->template apply_policy< typename policy_type::work_tag >(); + + task_root_type::template apply_single< functor_type , ResultType >( t ); + } + + TaskForEach( const function_dealloc_type arg_dealloc + , const int arg_sizeof_derived + , const int arg_dependence_capacity + , const policy_type & arg_policy + , const functor_type & arg_functor + ) + : task_base_type( arg_dealloc + , & apply_parallel + , arg_sizeof_derived + , arg_dependence_capacity + , arg_functor ) + , m_policy( arg_policy ) + {} + + TaskForEach() /* = delete */ ; + TaskForEach( const TaskForEach & ) /* = delete */ ; + TaskForEach & operator = ( const TaskForEach & ) /* = delete */ ; +}; + +//---------------------------------------------------------------------------- +/** \brief Reduce task in the Qthread execution space + * + * Derived from TaskMember< Kokkos::Qthread , ResultType , FunctorType > + * so that Functor can be cast to task root type without knowing policy. + */ +template< class Arg0 , class Arg1 , class Arg2 , class ResultType , class FunctorType > +class TaskReduce< Kokkos::RangePolicy< Arg0 , Arg1 , Arg2 , Kokkos::Qthread > + , ResultType + , FunctorType > + : TaskMember< Kokkos::Qthread , ResultType , FunctorType > +{ +public: + + typedef FunctorType functor_type ; + typedef RangePolicy< Arg0 , Arg1 , Arg2 , Kokkos::Qthread > policy_type ; + +private: + + friend class Kokkos::TaskPolicy< Kokkos::Qthread > ; + friend class Kokkos::Impl::TaskMember< Kokkos::Qthread , void , void > ; + + typedef TaskMember< Kokkos::Qthread , void , void > task_root_type ; + typedef TaskMember< Kokkos::Qthread , ResultType , FunctorType > task_base_type ; + typedef task_root_type::function_dealloc_type function_dealloc_type ; + + policy_type m_policy ; + + template< class Tag > + inline + void apply_policy( typename Impl::enable_if< Impl::is_same::value , ResultType & >::type result ) const + { + Impl::FunctorValueInit< functor_type , Tag >::init( *this , & result ); + const typename policy_type::member_type e = m_policy.end(); + for ( typename policy_type::member_type i = m_policy.begin() ; i < e ; ++i ) { + functor_type::operator()( i, result ); + } + } + + template< class Tag > + inline + void apply_policy( typename Impl::enable_if< ! Impl::is_same::value , ResultType & >::type result ) const + { + Impl::FunctorValueInit< functor_type , Tag >::init( *this , & result ); + const Tag tag ; + const typename policy_type::member_type e = m_policy.end(); + for ( typename policy_type::member_type i = m_policy.begin() ; i < e ; ++i ) { + functor_type::operator()( tag, i, result ); + } + } + + static + void apply_parallel( task_root_type * t ) + { + TaskReduce * const task = static_cast(t); + + task->template apply_policy< typename policy_type::work_tag >( task->task_base_type::m_result ); + + task_root_type::template apply_single< functor_type , ResultType >( t ); + } + + TaskReduce( const function_dealloc_type arg_dealloc + , const int arg_sizeof_derived + , const int arg_dependence_capacity + , const policy_type & arg_policy + , const functor_type & arg_functor + ) + : task_base_type( arg_dealloc + , & apply_parallel + , arg_sizeof_derived + , arg_dependence_capacity + , arg_functor ) + , m_policy( arg_policy ) + {} + + TaskReduce() /* = delete */ ; + TaskReduce( const TaskReduce & ) /* = delete */ ; + TaskReduce & operator = ( const TaskReduce & ) /* = delete */ ; +}; + + +} /* namespace Impl */ +} /* namespace Kokkos */ + +//---------------------------------------------------------------------------- +//---------------------------------------------------------------------------- + +namespace Kokkos { + +template<> +class TaskPolicy< Kokkos::Qthread > +{ +public: + + typedef Kokkos::Qthread execution_space ; + +private: + + typedef Impl::TaskMember< execution_space , void , void > task_root_type ; + + TaskPolicy & operator = ( const TaskPolicy & ) /* = delete */ ; + + template< class FunctorType > + static inline + const task_root_type * get_task_root( const FunctorType * f ) + { + typedef Impl::TaskMember< execution_space , typename FunctorType::value_type , FunctorType > task_type ; + return static_cast< const task_root_type * >( static_cast< const task_type * >(f) ); + } + + template< class FunctorType > + static inline + task_root_type * get_task_root( FunctorType * f ) + { + typedef Impl::TaskMember< execution_space , typename FunctorType::value_type , FunctorType > task_type ; + return static_cast< task_root_type * >( static_cast< task_type * >(f) ); + } + + const unsigned m_default_dependence_capacity ; + +public: + + KOKKOS_INLINE_FUNCTION + TaskPolicy() : m_default_dependence_capacity(4) {} + + KOKKOS_INLINE_FUNCTION + TaskPolicy( const TaskPolicy & rhs ) : m_default_dependence_capacity( rhs.m_default_dependence_capacity ) {} + + KOKKOS_INLINE_FUNCTION + explicit + TaskPolicy( const unsigned arg_default_dependence_capacity ) + : m_default_dependence_capacity( arg_default_dependence_capacity ) {} + + KOKKOS_INLINE_FUNCTION + TaskPolicy( const TaskPolicy & + , const unsigned arg_default_dependence_capacity ) + : m_default_dependence_capacity( arg_default_dependence_capacity ) {} + + //---------------------------------------- + + template< class ValueType > + const Future< ValueType , execution_space > & + spawn( const Future< ValueType , execution_space > & f ) const + { +#if defined( KOKKOS_ACTIVE_EXECUTION_MEMORY_SPACE_HOST ) + f.m_task->schedule(); +#endif + return f ; + } + + // Create single-thread task + + template< class FunctorType > + Future< typename FunctorType::value_type , execution_space > + create( const FunctorType & functor + , const unsigned dependence_capacity = ~0u ) const + { + typedef typename FunctorType::value_type value_type ; + typedef Impl::TaskMember< execution_space , value_type , FunctorType > task_type ; + return Future< value_type , execution_space >( +#if defined( KOKKOS_ACTIVE_EXECUTION_MEMORY_SPACE_HOST ) + task_root_type::create< task_type >( + functor , ( ~0u == dependence_capacity ? m_default_dependence_capacity : dependence_capacity ) ) +#endif + ); + } + + // Create parallel foreach task + + template< class PolicyType , class FunctorType > + Future< typename FunctorType::value_type , execution_space > + create_foreach( const PolicyType & policy + , const FunctorType & functor + , const unsigned dependence_capacity = ~0u ) const + { + typedef typename FunctorType::value_type value_type ; + typedef Impl::TaskForEach< PolicyType , value_type , FunctorType > task_type ; + return Future< value_type , execution_space >( +#if defined( KOKKOS_ACTIVE_EXECUTION_MEMORY_SPACE_HOST ) + task_root_type::create< task_type >( policy , functor , + ( ~0u == dependence_capacity ? m_default_dependence_capacity : dependence_capacity ) ) +#endif + ); + } + + // Create parallel reduce task + + template< class PolicyType , class FunctorType > + Future< typename FunctorType::value_type , execution_space > + create_reduce( const PolicyType & policy + , const FunctorType & functor + , const unsigned dependence_capacity = ~0u ) const + { + typedef typename FunctorType::value_type value_type ; + typedef Impl::TaskReduce< PolicyType , value_type , FunctorType > task_type ; + return Future< value_type , execution_space >( +#if defined( KOKKOS_ACTIVE_EXECUTION_MEMORY_SPACE_HOST ) + task_root_type::create< task_type >( policy , functor , + ( ~0u == dependence_capacity ? m_default_dependence_capacity : dependence_capacity ) ) +#endif + ); + } + + // Add dependence + template< class A1 , class A2 , class A3 , class A4 > + void add_dependence( const Future & after + , const Future & before + , typename Impl::enable_if + < Impl::is_same< typename Future::execution_space , execution_space >::value + && + Impl::is_same< typename Future::execution_space , execution_space >::value + >::type * = 0 + ) + { +#if defined( KOKKOS_ACTIVE_EXECUTION_MEMORY_SPACE_HOST ) + after.m_task->add_dependence( before.m_task ); +#endif + } + + //---------------------------------------- + // Functions for an executing task functor to query dependences, + // set new dependences, and respawn itself. + + template< class FunctorType > + Future< void , execution_space > + get_dependence( const FunctorType * task_functor , int i ) const + { + return Future( +#if defined( KOKKOS_ACTIVE_EXECUTION_MEMORY_SPACE_HOST ) + get_task_root(task_functor)->get_dependence(i) +#endif + ); + } + + template< class FunctorType > + int get_dependence( const FunctorType * task_functor ) const +#if defined( KOKKOS_ACTIVE_EXECUTION_MEMORY_SPACE_HOST ) + { return get_task_root(task_functor)->get_dependence(); } +#else + { return 0 ; } +#endif + + template< class FunctorType > + void clear_dependence( FunctorType * task_functor ) const + { +#if defined( KOKKOS_ACTIVE_EXECUTION_MEMORY_SPACE_HOST ) + get_task_root(task_functor)->clear_dependence(); +#endif + } + + template< class FunctorType , class A3 , class A4 > + void add_dependence( FunctorType * task_functor + , const Future & before + , typename Impl::enable_if + < Impl::is_same< typename Future::execution_space , execution_space >::value + >::type * = 0 + ) + { +#if defined( KOKKOS_ACTIVE_EXECUTION_MEMORY_SPACE_HOST ) + get_task_root(task_functor)->add_dependence( before.m_task ); +#endif + } + + template< class FunctorType > + void respawn( FunctorType * task_functor ) const +#if defined( KOKKOS_ACTIVE_EXECUTION_MEMORY_SPACE_HOST ) + { get_task_root(task_functor)->schedule(); } +#else + {} +#endif + +}; + +inline +void wait( TaskPolicy< Kokkos::Qthread > & ); + +inline +void wait( const Future< void , Kokkos::Qthread > & future ) +{ Impl::TaskMember< Kokkos::Qthread , void , void >::wait( future ); } + +} // namespace Kokkos + +//---------------------------------------------------------------------------- +//---------------------------------------------------------------------------- + +#endif /* #define KOKKOS_QTHREAD_TASK_HPP */ + diff --git a/lib/kokkos/core/src/Threads/Kokkos_ThreadsExec.cpp b/lib/kokkos/core/src/Threads/Kokkos_ThreadsExec.cpp new file mode 100755 index 0000000000..1c2db5f1a9 --- /dev/null +++ b/lib/kokkos/core/src/Threads/Kokkos_ThreadsExec.cpp @@ -0,0 +1,745 @@ +/* +//@HEADER +// ************************************************************************ +// +// Kokkos: Manycore Performance-Portable Multidimensional Arrays +// Copyright (2012) Sandia Corporation +// +// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +// the U.S. Government retains certain rights in this software. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// 1. Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// 3. Neither the name of the Corporation nor the names of the +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Questions? Contact H. Carter Edwards (hcedwar@sandia.gov) +// +// ************************************************************************ +//@HEADER +*/ + +#include + +#if defined( KOKKOS_HAVE_PTHREAD ) || defined( KOKKOS_HAVE_WINTHREAD ) + +#include +#include +#include +#include +#include +#include +#include +#include +#include + + +//---------------------------------------------------------------------------- +//---------------------------------------------------------------------------- + +namespace Kokkos { +namespace Impl { +namespace { + +ThreadsExec s_threads_process ; +ThreadsExec * s_threads_exec[ ThreadsExec::MAX_THREAD_COUNT ] = { 0 }; +pthread_t s_threads_pid[ ThreadsExec::MAX_THREAD_COUNT ] = { 0 }; +std::pair s_threads_coord[ ThreadsExec::MAX_THREAD_COUNT ]; + +int s_thread_pool_size[3] = { 0 , 0 , 0 }; + +unsigned s_current_reduce_size = 0 ; +unsigned s_current_shared_size = 0 ; + +void (* volatile s_current_function)( ThreadsExec & , const void * ); +const void * volatile s_current_function_arg = 0 ; + +struct Sentinel { + Sentinel() + { + HostSpace::register_in_parallel( ThreadsExec::in_parallel ); + } + + ~Sentinel() + { + if ( s_thread_pool_size[0] || + s_thread_pool_size[1] || + s_thread_pool_size[2] || + s_current_reduce_size || + s_current_shared_size || + s_current_function || + s_current_function_arg || + s_threads_exec[0] ) { + std::cerr << "ERROR : Process exiting without calling Kokkos::Threads::terminate()" << std::endl ; + } + } +}; + +inline +unsigned fan_size( const unsigned rank , const unsigned size ) +{ + const unsigned rank_rev = size - ( rank + 1 ); + unsigned count = 0 ; + for ( unsigned n = 1 ; ( rank_rev + n < size ) && ! ( rank_rev & n ) ; n <<= 1 ) { ++count ; } + return count ; +} + +} // namespace +} // namespace Impl +} // namespace Kokkos + +//---------------------------------------------------------------------------- +//---------------------------------------------------------------------------- + +namespace Kokkos { +namespace Impl { + +void execute_function_noop( ThreadsExec & , const void * ) {} + +void ThreadsExec::driver(void) +{ + ThreadsExec this_thread ; + + while ( ThreadsExec::Active == this_thread.m_pool_state ) { + + (*s_current_function)( this_thread , s_current_function_arg ); + + // Deactivate thread and wait for reactivation + this_thread.m_pool_state = ThreadsExec::Inactive ; + + wait_yield( this_thread.m_pool_state , ThreadsExec::Inactive ); + } +} + +ThreadsExec::ThreadsExec() + : m_pool_base(0) + , m_scratch(0) + , m_scratch_reduce_end(0) + , m_scratch_thread_end(0) + , m_pool_rank(0) + , m_pool_size(0) + , m_pool_fan_size(0) + , m_pool_state( ThreadsExec::Terminating ) +{ + if ( & s_threads_process != this ) { + + // A spawned thread + + ThreadsExec * const nil = 0 ; + + // Which entry in 's_threads_exec', possibly determined from hwloc binding + const int entry = ((size_t)s_current_function_arg) < size_t(s_thread_pool_size[0]) + ? ((size_t)s_current_function_arg) + : size_t(Kokkos::hwloc::bind_this_thread( s_thread_pool_size[0] , s_threads_coord )); + + // Given a good entry set this thread in the 's_threads_exec' array + if ( entry < s_thread_pool_size[0] && + nil == atomic_compare_exchange( s_threads_exec + entry , nil , this ) ) { + + m_pool_base = s_threads_exec ; + m_pool_rank = s_thread_pool_size[0] - ( entry + 1 ); + m_pool_size = s_thread_pool_size[0] ; + m_pool_fan_size = fan_size( m_pool_rank , m_pool_size ); + m_pool_state = ThreadsExec::Active ; + + s_threads_pid[ m_pool_rank ] = pthread_self(); + + // Inform spawning process that the threads_exec entry has been set. + s_threads_process.m_pool_state = ThreadsExec::Active ; + } + else { + // Inform spawning process that the threads_exec entry could not be set. + s_threads_process.m_pool_state = ThreadsExec::Terminating ; + } + } + else { + // Enables 'parallel_for' to execute on unitialized Threads device + m_pool_rank = 0 ; + m_pool_size = 1 ; + m_pool_state = ThreadsExec::Inactive ; + + s_threads_pid[ m_pool_rank ] = pthread_self(); + } +} + +ThreadsExec::~ThreadsExec() +{ + const unsigned entry = m_pool_size - ( m_pool_rank + 1 ); + + m_pool_base = 0 ; + m_scratch = 0 ; + m_scratch_reduce_end = 0 ; + m_scratch_thread_end = 0 ; + m_pool_rank = 0 ; + m_pool_size = 0 ; + m_pool_fan_size = 0 ; + + m_pool_state = ThreadsExec::Terminating ; + + if ( & s_threads_process != this && entry < MAX_THREAD_COUNT ) { + ThreadsExec * const nil = 0 ; + + atomic_compare_exchange( s_threads_exec + entry , this , nil ); + + s_threads_process.m_pool_state = ThreadsExec::Terminating ; + } +} + + +int ThreadsExec::get_thread_count() +{ + return s_thread_pool_size[0] ; +} + +ThreadsExec * ThreadsExec::get_thread( const int init_thread_rank ) +{ + ThreadsExec * const th = + init_thread_rank < s_thread_pool_size[0] + ? s_threads_exec[ s_thread_pool_size[0] - ( init_thread_rank + 1 ) ] : 0 ; + + if ( 0 == th || th->m_pool_rank != init_thread_rank ) { + std::ostringstream msg ; + msg << "Kokkos::Impl::ThreadsExec::get_thread ERROR : " + << "thread " << init_thread_rank << " of " << s_thread_pool_size[0] ; + if ( 0 == th ) { + msg << " does not exist" ; + } + else { + msg << " has wrong thread_rank " << th->m_pool_rank ; + } + Kokkos::Impl::throw_runtime_exception( msg.str() ); + } + + return th ; +} + +//---------------------------------------------------------------------------- + +void ThreadsExec::execute_get_binding( ThreadsExec & exec , const void * ) +{ + s_threads_coord[ exec.m_pool_rank ] = Kokkos::hwloc::get_this_thread_coordinate(); +} + +void ThreadsExec::execute_sleep( ThreadsExec & exec , const void * ) +{ + ThreadsExec::global_lock(); + ThreadsExec::global_unlock(); + + const int n = exec.m_pool_fan_size ; + const int rank_rev = exec.m_pool_size - ( exec.m_pool_rank + 1 ); + + for ( int i = 0 ; i < n ; ++i ) { + Impl::spinwait( exec.m_pool_base[ rank_rev + (1<m_pool_state , ThreadsExec::Active ); + } + + exec.m_pool_state = ThreadsExec::Inactive ; +} + +} +} + +//---------------------------------------------------------------------------- + +namespace Kokkos { +namespace Impl { + +void ThreadsExec::verify_is_process( const std::string & name , const bool initialized ) +{ + if ( ! is_process() ) { + std::string msg( name ); + msg.append( " FAILED : Called by a worker thread, can only be called by the master process." ); + Kokkos::Impl::throw_runtime_exception( msg ); + } + + if ( initialized && 0 == s_thread_pool_size[0] ) { + std::string msg( name ); + msg.append( " FAILED : Threads not initialized." ); + Kokkos::Impl::throw_runtime_exception( msg ); + } +} + +int ThreadsExec::in_parallel() +{ + // A thread function is in execution and + // the function argument is not the special threads process argument and + // the master process is a worker or is not the master process. + return s_current_function && + ( & s_threads_process != s_current_function_arg ) && + ( s_threads_process.m_pool_base || ! is_process() ); +} + +// Wait for root thread to become inactive +void ThreadsExec::fence() +{ + if ( s_thread_pool_size[0] ) { + // Wait for the root thread to complete: + Impl::spinwait( s_threads_exec[0]->m_pool_state , ThreadsExec::Active ); + } + + s_current_function = 0 ; + s_current_function_arg = 0 ; +} + +/** \brief Begin execution of the asynchronous functor */ +void ThreadsExec::start( void (*func)( ThreadsExec & , const void * ) , const void * arg ) +{ + verify_is_process("ThreadsExec::start" , true ); + + if ( s_current_function || s_current_function_arg ) { + Kokkos::Impl::throw_runtime_exception( std::string( "ThreadsExec::start() FAILED : already executing" ) ); + } + + s_current_function = func ; + s_current_function_arg = arg ; + + // Activate threads: + for ( int i = s_thread_pool_size[0] ; 0 < i-- ; ) { + s_threads_exec[i]->m_pool_state = ThreadsExec::Active ; + } + + if ( s_threads_process.m_pool_size ) { + // Master process is the root thread, run it: + (*func)( s_threads_process , arg ); + s_threads_process.m_pool_state = ThreadsExec::Inactive ; + } +} + +//---------------------------------------------------------------------------- + +bool ThreadsExec::sleep() +{ + verify_is_process("ThreadsExec::sleep", true ); + + if ( & execute_sleep == s_current_function ) return false ; + + fence(); + + ThreadsExec::global_lock(); + + s_current_function = & execute_sleep ; + + // Activate threads: + for ( unsigned i = s_thread_pool_size[0] ; 0 < i ; ) { + s_threads_exec[--i]->m_pool_state = ThreadsExec::Active ; + } + + return true ; +} + +bool ThreadsExec::wake() +{ + verify_is_process("ThreadsExec::wake", true ); + + if ( & execute_sleep != s_current_function ) return false ; + + ThreadsExec::global_unlock(); + + if ( s_threads_process.m_pool_base ) { + execute_sleep( s_threads_process , 0 ); + s_threads_process.m_pool_state = ThreadsExec::Inactive ; + } + + fence(); + + return true ; +} + +//---------------------------------------------------------------------------- + +void ThreadsExec::execute_serial( void (*func)( ThreadsExec & , const void * ) ) +{ + s_current_function = func ; + s_current_function_arg = & s_threads_process ; + + const unsigned begin = s_threads_process.m_pool_base ? 1 : 0 ; + + for ( unsigned i = s_thread_pool_size[0] ; begin < i ; ) { + ThreadsExec & th = * s_threads_exec[ --i ]; + + th.m_pool_state = ThreadsExec::Active ; + + wait_yield( th.m_pool_state , ThreadsExec::Active ); + } + + if ( s_threads_process.m_pool_base ) { + s_threads_process.m_pool_state = ThreadsExec::Active ; + (*func)( s_threads_process , 0 ); + s_threads_process.m_pool_state = ThreadsExec::Inactive ; + } + + s_current_function_arg = 0 ; + s_current_function = 0 ; +} + +//---------------------------------------------------------------------------- + +void * ThreadsExec::root_reduce_scratch() +{ + return s_threads_process.reduce_memory(); +} + +void ThreadsExec::execute_resize_scratch( ThreadsExec & exec , const void * ) +{ + if ( exec.m_scratch ) { + HostSpace::decrement( exec.m_scratch ); + exec.m_scratch = 0 ; + } + + exec.m_scratch_reduce_end = s_threads_process.m_scratch_reduce_end ; + exec.m_scratch_thread_end = s_threads_process.m_scratch_thread_end ; + + if ( s_threads_process.m_scratch_thread_end ) { + + exec.m_scratch = + HostSpace::allocate( "thread_scratch" , s_threads_process.m_scratch_thread_end ); + + unsigned * ptr = (unsigned *)( exec.m_scratch ); + unsigned * const end = ptr + s_threads_process.m_scratch_thread_end / sizeof(unsigned); + + // touch on this thread + while ( ptr < end ) *ptr++ = 0 ; + } +} + +void * ThreadsExec::resize_scratch( size_t reduce_size , size_t thread_size ) +{ + enum { ALIGN_MASK = Kokkos::Impl::MEMORY_ALIGNMENT - 1 }; + + fence(); + + const size_t old_reduce_size = s_threads_process.m_scratch_reduce_end ; + const size_t old_thread_size = s_threads_process.m_scratch_thread_end - s_threads_process.m_scratch_reduce_end ; + + reduce_size = ( reduce_size + ALIGN_MASK ) & ~ALIGN_MASK ; + thread_size = ( thread_size + ALIGN_MASK ) & ~ALIGN_MASK ; + + // Increase size or deallocate completely. + + if ( ( old_reduce_size < reduce_size ) || + ( old_thread_size < thread_size ) || + ( ( reduce_size == 0 && thread_size == 0 ) && + ( old_reduce_size != 0 || old_thread_size != 0 ) ) ) { + + verify_is_process( "ThreadsExec::resize_scratch" , true ); + + s_threads_process.m_scratch_reduce_end = reduce_size ; + s_threads_process.m_scratch_thread_end = reduce_size + thread_size ; + + execute_serial( & execute_resize_scratch ); + + s_threads_process.m_scratch = s_threads_exec[0]->m_scratch ; + } + + return s_threads_process.m_scratch ; +} + +//---------------------------------------------------------------------------- + +void ThreadsExec::print_configuration( std::ostream & s , const bool detail ) +{ + verify_is_process("ThreadsExec::print_configuration",false); + + fence(); + + const unsigned numa_count = Kokkos::hwloc::get_available_numa_count(); + const unsigned cores_per_numa = Kokkos::hwloc::get_available_cores_per_numa(); + const unsigned threads_per_core = Kokkos::hwloc::get_available_threads_per_core(); + + // Forestall compiler warnings for unused variables. + (void) numa_count; + (void) cores_per_numa; + (void) threads_per_core; + + s << "Kokkos::Threads" ; + +#if defined( KOKKOS_HAVE_PTHREAD ) + s << " KOKKOS_HAVE_PTHREAD" ; +#endif +#if defined( KOKKOS_HAVE_HWLOC ) + s << " hwloc[" << numa_count << "x" << cores_per_numa << "x" << threads_per_core << "]" ; +#endif + + if ( s_thread_pool_size[0] ) { + s << " threads[" << s_thread_pool_size[0] << "]" + << " threads_per_numa[" << s_thread_pool_size[1] << "]" + << " threads_per_core[" << s_thread_pool_size[2] << "]" + ; + if ( 0 == s_threads_process.m_pool_base ) { s << " Asynchronous" ; } + s << " ReduceScratch[" << s_current_reduce_size << "]" + << " SharedScratch[" << s_current_shared_size << "]" ; + s << std::endl ; + + if ( detail ) { + + execute_serial( & execute_get_binding ); + + for ( int i = 0 ; i < s_thread_pool_size[0] ; ++i ) { + ThreadsExec * const th = s_threads_exec[i] ; + s << " Thread hwloc(" + << s_threads_coord[i].first << "." + << s_threads_coord[i].second << ")" ; + + s_threads_coord[i].first = ~0u ; + s_threads_coord[i].second = ~0u ; + + if ( th ) { + const int rank_rev = th->m_pool_size - ( th->m_pool_rank + 1 ); + + s << " rank(" << th->m_pool_rank << ")" ; + + if ( th->m_pool_fan_size ) { + s << " Fan{" ; + for ( int j = 0 ; j < th->m_pool_fan_size ; ++j ) { + s << " " << th->m_pool_base[rank_rev+(1<m_pool_rank ; + } + s << " }" ; + } + + if ( th == & s_threads_process ) { + s << " is_process" ; + } + } + s << std::endl ; + } + } + } + else { + s << " not initialized" << std::endl ; + } +} + +//---------------------------------------------------------------------------- + +int ThreadsExec::is_initialized() +{ return 0 != s_threads_exec[0] ; } + +void ThreadsExec::initialize( unsigned thread_count , + unsigned use_numa_count , + unsigned use_cores_per_numa , + bool allow_asynchronous_threadpool ) +{ + static const Sentinel sentinel ; + + const bool is_initialized = 0 != s_thread_pool_size[0] ; + + unsigned thread_spawn_failed = 0 ; + + for ( int i = 0; i < ThreadsExec::MAX_THREAD_COUNT ; i++) + s_threads_exec[i] = NULL; + + if ( ! is_initialized ) { + + // If thread_count, use_numa_count, or use_cores_per_numa are zero + // then they will be given default values based upon hwloc detection + // and allowed asynchronous execution. + + const bool hwloc_avail = hwloc::available(); + + const unsigned thread_spawn_begin = + hwloc::thread_mapping( "Kokkos::Threads::initialize" , + allow_asynchronous_threadpool , + thread_count , + use_numa_count , + use_cores_per_numa , + s_threads_coord ); + + const std::pair proc_coord = s_threads_coord[0] ; + + if ( thread_spawn_begin ) { + // Synchronous with s_threads_coord[0] as the process core + // Claim entry #0 for binding the process core. + s_threads_coord[0] = std::pair(~0u,~0u); + } + + s_thread_pool_size[0] = thread_count ; + s_thread_pool_size[1] = s_thread_pool_size[0] / use_numa_count ; + s_thread_pool_size[2] = s_thread_pool_size[1] / use_cores_per_numa ; + s_current_function = & execute_function_noop ; // Initialization work function + + for ( unsigned ith = thread_spawn_begin ; ith < thread_count ; ++ith ) { + + s_threads_process.m_pool_state = ThreadsExec::Inactive ; + + // If hwloc available then spawned thread will + // choose its own entry in 's_threads_coord' + // otherwise specify the entry. + s_current_function_arg = (void*)static_cast( hwloc_avail ? ~0u : ith ); + + // Spawn thread executing the 'driver()' function. + // Wait until spawned thread has attempted to initialize. + // If spawning and initialization is successfull then + // an entry in 's_threads_exec' will be assigned. + if ( ThreadsExec::spawn() ) { + wait_yield( s_threads_process.m_pool_state , ThreadsExec::Inactive ); + } + if ( s_threads_process.m_pool_state == ThreadsExec::Terminating ) break ; + } + + // Wait for all spawned threads to deactivate before zeroing the function. + + for ( unsigned ith = thread_spawn_begin ; ith < thread_count ; ++ith ) { + // Try to protect against cache coherency failure by casting to volatile. + ThreadsExec * const th = ((ThreadsExec * volatile *)s_threads_exec)[ith] ; + if ( th ) { + wait_yield( th->m_pool_state , ThreadsExec::Active ); + } + else { + ++thread_spawn_failed ; + } + } + + s_current_function = 0 ; + s_current_function_arg = 0 ; + s_threads_process.m_pool_state = ThreadsExec::Inactive ; + + if ( ! thread_spawn_failed ) { + // Bind process to the core on which it was located before spawning occured + Kokkos::hwloc::bind_this_thread( proc_coord ); + + if ( thread_spawn_begin ) { // Include process in pool. + s_threads_exec[0] = & s_threads_process ; + s_threads_process.m_pool_base = s_threads_exec ; + s_threads_process.m_pool_rank = thread_count - 1 ; // Reversed for scan-compatible reductions + s_threads_process.m_pool_size = thread_count ; + s_threads_process.m_pool_fan_size = fan_size( s_threads_process.m_pool_rank , s_threads_process.m_pool_size ); + s_threads_pid[ s_threads_process.m_pool_rank ] = pthread_self(); + } + else { + s_threads_process.m_pool_base = 0 ; + s_threads_process.m_pool_rank = 0 ; + s_threads_process.m_pool_size = 0 ; + s_threads_process.m_pool_fan_size = 0 ; + } + + // Initial allocations: + ThreadsExec::resize_scratch( 1024 , 1024 ); + } + else { + s_thread_pool_size[0] = 0 ; + s_thread_pool_size[1] = 0 ; + s_thread_pool_size[2] = 0 ; + } + } + + if ( is_initialized || thread_spawn_failed ) { + + std::ostringstream msg ; + + msg << "Kokkos::Threads::initialize ERROR" ; + + if ( is_initialized ) { + msg << " : already initialized" ; + } + if ( thread_spawn_failed ) { + msg << " : failed to spawn " << thread_spawn_failed << " threads" ; + } + + Kokkos::Impl::throw_runtime_exception( msg.str() ); + } +} + +//---------------------------------------------------------------------------- + +void ThreadsExec::finalize() +{ + verify_is_process("ThreadsExec::finalize",false); + + fence(); + + resize_scratch(0,0); + + const unsigned begin = s_threads_process.m_pool_base ? 1 : 0 ; + + for ( unsigned i = s_thread_pool_size[0] ; begin < i-- ; ) { + + if ( s_threads_exec[i] ) { + + s_threads_exec[i]->m_pool_state = ThreadsExec::Terminating ; + + wait_yield( s_threads_process.m_pool_state , ThreadsExec::Inactive ); + + s_threads_process.m_pool_state = ThreadsExec::Inactive ; + } + + s_threads_pid[i] = 0 ; + } + + if ( s_threads_process.m_pool_base ) { + ( & s_threads_process )->~ThreadsExec(); + s_threads_exec[0] = 0 ; + } + + Kokkos::hwloc::unbind_this_thread(); + + s_thread_pool_size[0] = 0 ; + s_thread_pool_size[1] = 0 ; + s_thread_pool_size[2] = 0 ; + + // Reset master thread to run solo. + s_threads_process.m_pool_base = 0 ; + s_threads_process.m_pool_rank = 0 ; + s_threads_process.m_pool_size = 1 ; + s_threads_process.m_pool_fan_size = 0 ; + s_threads_process.m_pool_state = ThreadsExec::Inactive ; +} + +//---------------------------------------------------------------------------- + +} /* namespace Impl */ +} /* namespace Kokkos */ + +//---------------------------------------------------------------------------- +//---------------------------------------------------------------------------- + +namespace Kokkos { + +Threads & Threads::instance(int) +{ + static Threads t ; + return t ; +} + +int Threads::thread_pool_size( int depth ) +{ + return Impl::s_thread_pool_size[depth]; +} + +#if defined( KOKKOS_ACTIVE_EXECUTION_MEMORY_SPACE_HOST ) +int Threads::thread_pool_rank() +{ + const pthread_t pid = pthread_self(); + int i = 0; + while ( ( i < Impl::s_thread_pool_size[0] ) && ( pid != Impl::s_threads_pid[i] ) ) { ++i ; } + return i ; +} +#endif + +} /* namespace Kokkos */ + +//---------------------------------------------------------------------------- +//---------------------------------------------------------------------------- + +#endif /* #if defined( KOKKOS_HAVE_PTHREAD ) || defined( KOKKOS_HAVE_WINTHREAD ) */ + diff --git a/lib/kokkos/core/src/Threads/Kokkos_ThreadsExec.hpp b/lib/kokkos/core/src/Threads/Kokkos_ThreadsExec.hpp new file mode 100755 index 0000000000..e60a1094ad --- /dev/null +++ b/lib/kokkos/core/src/Threads/Kokkos_ThreadsExec.hpp @@ -0,0 +1,1041 @@ +/* +//@HEADER +// ************************************************************************ +// +// Kokkos: Manycore Performance-Portable Multidimensional Arrays +// Copyright (2012) Sandia Corporation +// +// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +// the U.S. Government retains certain rights in this software. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// 1. Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// 3. Neither the name of the Corporation nor the names of the +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Questions? Contact H. Carter Edwards (hcedwar@sandia.gov) +// +// ************************************************************************ +//@HEADER +*/ + +#ifndef KOKKOS_THREADSEXEC_HPP +#define KOKKOS_THREADSEXEC_HPP + +#include + +#include +#include +#include + +#include + +//---------------------------------------------------------------------------- + +namespace Kokkos { +namespace Impl { + +//---------------------------------------------------------------------------- + +template< class > struct ThreadsExecAdapter ; + +//---------------------------------------------------------------------------- + +class ThreadsExecTeamMember ; + +class ThreadsExec { +public: + + // Fan array has log_2(NT) reduction threads plus 2 scan threads + // Currently limited to 16k threads. + enum { MAX_FAN_COUNT = 16 }; + enum { MAX_THREAD_COUNT = 1 << ( MAX_FAN_COUNT - 2 ) }; + enum { VECTOR_LENGTH = 8 }; + + /** \brief States of a worker thread */ + enum { Terminating ///< Termination in progress + , Inactive ///< Exists, waiting for work + , Active ///< Exists, performing work + , Rendezvous ///< Exists, waiting in a barrier or reduce + + , ScanCompleted + , ScanAvailable + , ReductionAvailable + }; + +private: + + friend class ThreadsExecTeamMember ; + friend class ThreadsExecTeamVectorMember ; + friend class Kokkos::Threads ; + + // Fan-in operations' root is the highest ranking thread + // to place the 'scan' reduction intermediate values on + // the threads that need them. + // For a simple reduction the thread location is arbitrary. + + /** \brief Reduction memory reserved for team reductions */ + enum { REDUCE_TEAM_BASE = 512 }; + + ThreadsExec * const * m_pool_base ; ///< Base for pool fan-in + + void * m_scratch ; + int m_scratch_reduce_end ; + int m_scratch_thread_end ; + int m_pool_rank ; + int m_pool_size ; + int m_pool_fan_size ; + int volatile m_pool_state ; ///< State for global synchronizations + + + static void global_lock(); + static void global_unlock(); + static bool spawn(); + + static void execute_resize_scratch( ThreadsExec & , const void * ); + static void execute_sleep( ThreadsExec & , const void * ); + static void execute_get_binding( ThreadsExec & , const void * ); + + ThreadsExec( const ThreadsExec & ); + ThreadsExec & operator = ( const ThreadsExec & ); + + static void execute_serial( void (*)( ThreadsExec & , const void * ) ); + +public: + + KOKKOS_INLINE_FUNCTION int pool_size() const { return m_pool_size ; } + KOKKOS_INLINE_FUNCTION int pool_rank() const { return m_pool_rank ; } + + static int get_thread_count(); + static ThreadsExec * get_thread( const int init_thread_rank ); + + inline void * reduce_memory() const { return ((unsigned char *) m_scratch ); } + KOKKOS_INLINE_FUNCTION void * scratch_memory() const { return ((unsigned char *) m_scratch ) + m_scratch_reduce_end ; } + + static void driver(void); + + ~ThreadsExec(); + ThreadsExec(); + + static void * resize_scratch( size_t reduce_size , size_t thread_size ); + + static void * root_reduce_scratch(); + + static bool is_process(); + + static void verify_is_process( const std::string & , const bool initialized ); + + static int is_initialized(); + + static void initialize( unsigned thread_count , + unsigned use_numa_count , + unsigned use_cores_per_numa , + bool allow_asynchronous_threadpool ); + + static void finalize(); + + /* Given a requested team size, return valid team size */ + static unsigned team_size_valid( unsigned ); + + static void print_configuration( std::ostream & , const bool detail = false ); + + //------------------------------------ + + static void wait_yield( volatile int & , const int ); + + //------------------------------------ + // All-thread functions: + + template< class FunctorType , class ArgTag > + inline + void fan_in_reduce( const FunctorType & f ) const + { + typedef Kokkos::Impl::FunctorValueJoin< FunctorType , ArgTag > Join ; + typedef Kokkos::Impl::FunctorFinal< FunctorType , ArgTag > Final ; + + const int rev_rank = m_pool_size - ( m_pool_rank + 1 ); + + for ( int i = 0 ; i < m_pool_fan_size ; ++i ) { + + ThreadsExec & fan = *m_pool_base[ rev_rank + ( 1 << i ) ] ; + + Impl::spinwait( fan.m_pool_state , ThreadsExec::Active ); + + Join::join( f , reduce_memory() , fan.reduce_memory() ); + } + + if ( ! rev_rank ) { + Final::final( f , reduce_memory() ); + } + } + + inline + void fan_in() const + { + const int rev_rank = m_pool_size - ( m_pool_rank + 1 ); + + for ( int i = 0 ; i < m_pool_fan_size ; ++i ) { + Impl::spinwait( m_pool_base[rev_rank+(1<m_pool_state , ThreadsExec::Active ); + } + } + + template< class FunctorType , class ArgTag > + inline + void scan_large( const FunctorType & f ) + { + // Sequence of states: + // 0) Active : entry and exit state + // 1) ReductionAvailable : reduction value available + // 2) ScanAvailable : inclusive scan value available + // 3) Rendezvous : All threads inclusive scan value are available + // 4) ScanCompleted : exclusive scan value copied + + typedef Kokkos::Impl::FunctorValueTraits< FunctorType , ArgTag > Traits ; + typedef Kokkos::Impl::FunctorValueJoin< FunctorType , ArgTag > Join ; + typedef Kokkos::Impl::FunctorValueInit< FunctorType , ArgTag > Init ; + + typedef typename Traits::value_type scalar_type ; + + const int rev_rank = m_pool_size - ( m_pool_rank + 1 ); + const unsigned count = Traits::value_count( f ); + + scalar_type * const work_value = (scalar_type *) reduce_memory(); + + //-------------------------------- + // Fan-in reduction with highest ranking thread as the root + for ( int i = 0 ; i < m_pool_fan_size ; ++i ) { + ThreadsExec & fan = *m_pool_base[ rev_rank + (1< ReductionAvailable (or ScanAvailable) + Impl::spinwait( fan.m_pool_state , ThreadsExec::Active ); + Join::join( f , work_value , fan.reduce_memory() ); + } + + // Copy reduction value to scan value before releasing from this phase. + for ( unsigned i = 0 ; i < count ; ++i ) { work_value[i+count] = work_value[i] ; } + + if ( rev_rank ) { + + // Set: Active -> ReductionAvailable + m_pool_state = ThreadsExec::ReductionAvailable ; + + // Wait for contributing threads' scan value to be available. + if ( ( 1 << m_pool_fan_size ) < ( m_pool_rank + 1 ) ) { + ThreadsExec & th = *m_pool_base[ rev_rank + ( 1 << m_pool_fan_size ) ] ; + + // Wait: Active -> ReductionAvailable + // Wait: ReductionAvailable -> ScanAvailable + Impl::spinwait( th.m_pool_state , ThreadsExec::Active ); + Impl::spinwait( th.m_pool_state , ThreadsExec::ReductionAvailable ); + + Join::join( f , work_value + count , ((scalar_type *)th.reduce_memory()) + count ); + } + + // This thread has completed inclusive scan + // Set: ReductionAvailable -> ScanAvailable + m_pool_state = ThreadsExec::ScanAvailable ; + + // Wait for all threads to complete inclusive scan + // Wait: ScanAvailable -> Rendezvous + Impl::spinwait( m_pool_state , ThreadsExec::ScanAvailable ); + } + + //-------------------------------- + + for ( int i = 0 ; i < m_pool_fan_size ; ++i ) { + ThreadsExec & fan = *m_pool_base[ rev_rank + (1< ScanAvailable + Impl::spinwait( fan.m_pool_state , ThreadsExec::ReductionAvailable ); + // Set: ScanAvailable -> Rendezvous + fan.m_pool_state = ThreadsExec::Rendezvous ; + } + + // All threads have completed the inclusive scan. + // All non-root threads are in the Rendezvous state. + // Threads are free to overwrite their reduction value. + //-------------------------------- + + if ( ( rev_rank + 1 ) < m_pool_size ) { + // Exclusive scan: copy the previous thread's inclusive scan value + + ThreadsExec & th = *m_pool_base[ rev_rank + 1 ] ; // Not the root thread + + const scalar_type * const src_value = ((scalar_type *)th.reduce_memory()) + count ; + + for ( unsigned j = 0 ; j < count ; ++j ) { work_value[j] = src_value[j]; } + } + else { + (void) Init::init( f , work_value ); + } + + //-------------------------------- + // Wait for all threads to copy previous thread's inclusive scan value + // Wait for all threads: Rendezvous -> ScanCompleted + for ( int i = 0 ; i < m_pool_fan_size ; ++i ) { + Impl::spinwait( m_pool_base[ rev_rank + (1<m_pool_state , ThreadsExec::Rendezvous ); + } + if ( rev_rank ) { + // Set: ScanAvailable -> ScanCompleted + m_pool_state = ThreadsExec::ScanCompleted ; + // Wait: ScanCompleted -> Active + Impl::spinwait( m_pool_state , ThreadsExec::ScanCompleted ); + } + // Set: ScanCompleted -> Active + for ( int i = 0 ; i < m_pool_fan_size ; ++i ) { + m_pool_base[ rev_rank + (1<m_pool_state = ThreadsExec::Active ; + } + } + + template< class FunctorType , class ArgTag > + inline + void scan_small( const FunctorType & f ) + { + typedef Kokkos::Impl::FunctorValueTraits< FunctorType , ArgTag > Traits ; + typedef Kokkos::Impl::FunctorValueJoin< FunctorType , ArgTag > Join ; + typedef Kokkos::Impl::FunctorValueInit< FunctorType , ArgTag > Init ; + + typedef typename Traits::value_type scalar_type ; + + const int rev_rank = m_pool_size - ( m_pool_rank + 1 ); + const unsigned count = Traits::value_count( f ); + + scalar_type * const work_value = (scalar_type *) reduce_memory(); + + //-------------------------------- + // Fan-in reduction with highest ranking thread as the root + for ( int i = 0 ; i < m_pool_fan_size ; ++i ) { + // Wait: Active -> Rendezvous + Impl::spinwait( m_pool_base[ rev_rank + (1<m_pool_state , ThreadsExec::Active ); + } + + for ( unsigned i = 0 ; i < count ; ++i ) { work_value[i+count] = work_value[i]; } + + if ( rev_rank ) { + m_pool_state = ThreadsExec::Rendezvous ; + // Wait: Rendezvous -> Active + Impl::spinwait( m_pool_state , ThreadsExec::Rendezvous ); + } + else { + // Root thread does the thread-scan before releasing threads + + scalar_type * ptr_prev = 0 ; + + for ( int rank = 0 ; rank < m_pool_size ; ++rank ) { + scalar_type * const ptr = (scalar_type *) get_thread( rank )->reduce_memory(); + if ( rank ) { + for ( unsigned i = 0 ; i < count ; ++i ) { ptr[i] = ptr_prev[ i + count ]; } + Join::join( f , ptr + count , ptr ); + } + else { + (void) Init::init( f , ptr ); + } + ptr_prev = ptr ; + } + } + + for ( int i = 0 ; i < m_pool_fan_size ; ++i ) { + m_pool_base[ rev_rank + (1<m_pool_state = ThreadsExec::Active ; + } + } + + //------------------------------------ + /** \brief Wait for previous asynchronous functor to + * complete and release the Threads device. + * Acquire the Threads device and start this functor. + */ + static void start( void (*)( ThreadsExec & , const void * ) , const void * ); + + static int in_parallel(); + static void fence(); + static bool sleep(); + static bool wake(); +}; + +//---------------------------------------------------------------------------- +//---------------------------------------------------------------------------- + +class ThreadsExecTeamMember { +private: + + enum { TEAM_REDUCE_SIZE = 512 }; + + typedef Kokkos::Threads execution_space ; + typedef execution_space::scratch_memory_space space ; + + Impl::ThreadsExec & m_exec ; + space m_team_shared ; + ThreadsExec * const * m_team_base ; ///< Base for team fan-in + int m_team_shared_size ; + int m_team_size ; + int m_team_rank ; + int m_team_rank_rev ; + int m_league_size ; + int m_league_end ; + int m_league_rank ; + + inline + void set_team_shared() + { new( & m_team_shared ) space( ((char *) (*m_team_base)->scratch_memory()) + TEAM_REDUCE_SIZE , m_team_shared_size ); } + + // Fan-in and wait until the matching fan-out is called. + // The root thread which does not wait will return true. + // All other threads will return false during the fan-out. + KOKKOS_INLINE_FUNCTION bool team_fan_in() const + { + int n , j ; + + // Wait for fan-in threads + for ( n = 1 ; ( ! ( m_team_rank_rev & n ) ) && ( ( j = m_team_rank_rev + n ) < m_team_size ) ; n <<= 1 ) { + Impl::spinwait( m_team_base[j]->m_pool_state , ThreadsExec::Active ); + } + + // If not root then wait for release + if ( m_team_rank_rev ) { + m_exec.m_pool_state = ThreadsExec::Rendezvous ; + Impl::spinwait( m_exec.m_pool_state , ThreadsExec::Rendezvous ); + } + + return ! m_team_rank_rev ; + } + + KOKKOS_INLINE_FUNCTION void team_fan_out() const + { + int n , j ; + for ( n = 1 ; ( ! ( m_team_rank_rev & n ) ) && ( ( j = m_team_rank_rev + n ) < m_team_size ) ; n <<= 1 ) { + m_team_base[j]->m_pool_state = ThreadsExec::Active ; + } + } + +public: + + KOKKOS_INLINE_FUNCTION static int team_reduce_size() { return TEAM_REDUCE_SIZE ; } + + KOKKOS_INLINE_FUNCTION + const execution_space::scratch_memory_space & team_shmem() const + { return m_team_shared ; } + + KOKKOS_INLINE_FUNCTION int league_rank() const { return m_league_rank ; } + KOKKOS_INLINE_FUNCTION int league_size() const { return m_league_size ; } + KOKKOS_INLINE_FUNCTION int team_rank() const { return m_team_rank ; } + KOKKOS_INLINE_FUNCTION int team_size() const { return m_team_size ; } + + KOKKOS_INLINE_FUNCTION void team_barrier() const + { + team_fan_in(); + team_fan_out(); + } + + template + KOKKOS_INLINE_FUNCTION + void team_broadcast(ValueType& value, const int& thread_id) const + { +#if ! defined( KOKKOS_ACTIVE_EXECUTION_MEMORY_SPACE_HOST ) + { } +#else + // Make sure there is enough scratch space: + typedef typename if_c< sizeof(ValueType) < TEAM_REDUCE_SIZE + , ValueType , void >::type type ; + + type * const local_value = ((type*) m_exec.scratch_memory()); + if(team_rank() == thread_id) + *local_value = value; + memory_fence(); + team_barrier(); + value = *local_value; +#endif + } + + template< typename Type > + KOKKOS_INLINE_FUNCTION Type team_reduce( const Type & value ) const +#if ! defined( KOKKOS_ACTIVE_EXECUTION_MEMORY_SPACE_HOST ) + { return Type(); } +#else + { + // Make sure there is enough scratch space: + typedef typename if_c< sizeof(Type) < ThreadsExec::REDUCE_TEAM_BASE , Type , void >::type type ; + + *((volatile type*) m_exec.scratch_memory() ) = value ; + + memory_fence(); + + type & accum = *((type *) m_team_base[0]->scratch_memory() ); + + if ( team_fan_in() ) { + for ( int i = 1 ; i < m_team_size ; ++i ) { + accum += *((type *) m_team_base[i]->scratch_memory() ); + } + memory_fence(); + } + + team_fan_out(); + + return accum ; + } +#endif + +#ifdef KOKKOS_HAVE_CXX11 + template< class ValueType, class JoinOp > + KOKKOS_INLINE_FUNCTION ValueType + team_reduce( const ValueType & value + , const JoinOp & op_in ) const + #if ! defined( KOKKOS_ACTIVE_EXECUTION_MEMORY_SPACE_HOST ) + { return ValueType(); } + #else + { + typedef ValueType value_type; + const JoinLambdaAdapter op(op_in); + #endif +#else // KOKKOS_HAVE_CXX11 + template< class JoinOp > + KOKKOS_INLINE_FUNCTION typename JoinOp::value_type + team_reduce( const typename JoinOp::value_type & value + , const JoinOp & op ) const + #if ! defined( KOKKOS_ACTIVE_EXECUTION_MEMORY_SPACE_HOST ) + { return typename JoinOp::value_type(); } + #else + { + typedef typename JoinOp::value_type value_type; + #endif +#endif // KOKKOS_HAVE_CXX11 +#if defined( KOKKOS_ACTIVE_EXECUTION_MEMORY_SPACE_HOST ) + // Make sure there is enough scratch space: + typedef typename if_c< sizeof(value_type) < ThreadsExec::REDUCE_TEAM_BASE + , value_type , void >::type type ; + + type * const local_value = ((type*) m_exec.scratch_memory()); + + // Set this thread's contribution + *local_value = value ; + + // Fence to make sure the base team member has access: + memory_fence(); + + if ( team_fan_in() ) { + // The last thread to synchronize returns true, all other threads wait for team_fan_out() + type * const team_value = ((type*) m_team_base[0]->scratch_memory()); + + // Join to the team value: + for ( int i = 1 ; i < m_team_size ; ++i ) { + op.join( *team_value , *((type*) m_team_base[i]->scratch_memory()) ); + } + + // Team base thread may "lap" member threads so copy out to their local value. + for ( int i = 1 ; i < m_team_size ; ++i ) { + *((type*) m_team_base[i]->scratch_memory()) = *team_value ; + } + + // Fence to make sure all team members have access + memory_fence(); + } + + team_fan_out(); + + // Value was changed by the team base + return *((type volatile const *) local_value); + } +#endif + + /** \brief Intra-team exclusive prefix sum with team_rank() ordering + * with intra-team non-deterministic ordering accumulation. + * + * The global inter-team accumulation value will, at the end of the + * league's parallel execution, be the scan's total. + * Parallel execution ordering of the league's teams is non-deterministic. + * As such the base value for each team's scan operation is similarly + * non-deterministic. + */ + template< typename ArgType > + KOKKOS_INLINE_FUNCTION ArgType team_scan( const ArgType & value , ArgType * const global_accum ) const +#if ! defined( KOKKOS_ACTIVE_EXECUTION_MEMORY_SPACE_HOST ) + { return ArgType(); } +#else + { + // Make sure there is enough scratch space: + typedef typename if_c< sizeof(ArgType) < ThreadsExec::REDUCE_TEAM_BASE , ArgType , void >::type type ; + + volatile type * const work_value = ((type*) m_exec.scratch_memory()); + + *work_value = value ; + + memory_fence(); + + if ( team_fan_in() ) { + // The last thread to synchronize returns true, all other threads wait for team_fan_out() + // m_team_base[0] == highest ranking team member + // m_team_base[ m_team_size - 1 ] == lowest ranking team member + // + // 1) copy from lower to higher rank, initialize lowest rank to zero + // 2) prefix sum from lowest to highest rank, skipping lowest rank + + type accum = 0 ; + + if ( global_accum ) { + for ( int i = m_team_size ; i-- ; ) { + type & val = *((type*) m_team_base[i]->scratch_memory()); + accum += val ; + } + accum = atomic_fetch_add( global_accum , accum ); + } + + for ( int i = m_team_size ; i-- ; ) { + type & val = *((type*) m_team_base[i]->scratch_memory()); + const type offset = accum ; + accum += val ; + val = offset ; + } + + memory_fence(); + } + + team_fan_out(); + + return *work_value ; + } +#endif + + /** \brief Intra-team exclusive prefix sum with team_rank() ordering. + * + * The highest rank thread can compute the reduction total as + * reduction_total = dev.team_scan( value ) + value ; + */ + template< typename ArgType > + KOKKOS_INLINE_FUNCTION ArgType team_scan( const ArgType & value ) const + { return this-> template team_scan( value , 0 ); } + +#ifdef KOKKOS_HAVE_CXX11 + + /** \brief Inter-thread parallel for. Executes op(iType i) for each i=0..N-1. + * + * The range i=0..N-1 is mapped to all threads of the the calling thread team. + * This functionality requires C++11 support.*/ + template< typename iType, class Operation> + KOKKOS_INLINE_FUNCTION void team_par_for(const iType n, const Operation & op) const { + const int chunk = ((n+m_team_size-1)/m_team_size); + const int start = chunk*m_team_rank; + const int end = start+chunk + ThreadsExecTeamMember( Impl::ThreadsExec & exec + , const TeamPolicy< Arg0 , Arg1 , Kokkos::Threads > & team + , const int shared_size ) + : m_exec( exec ) + , m_team_shared(0,0) + , m_team_base(0) + , m_team_shared_size( shared_size ) + , m_team_size(0) + , m_team_rank(0) + , m_team_rank_rev(0) + , m_league_size(0) + , m_league_end(0) + , m_league_rank(0) + { + if ( team.league_size() ) { + // Execution is using device-team interface: + + const int pool_rank_rev = exec.pool_size() - ( exec.pool_rank() + 1 ); + const int team_rank_rev = pool_rank_rev % team.team_alloc(); + + // May be using fewer threads per team than a multiple of threads per core, + // some threads will idle. + + if ( team_rank_rev < team.team_size() ) { + const size_t pool_league_size = exec.pool_size() / team.team_alloc() ; + const size_t pool_league_rank_rev = pool_rank_rev / team.team_alloc() ; + const size_t pool_league_rank = pool_league_size - ( pool_league_rank_rev + 1 ); + + m_team_base = exec.m_pool_base + team.team_alloc() * pool_league_rank_rev ; + m_team_size = team.team_size() ; + m_team_rank = team.team_size() - ( team_rank_rev + 1 ); + m_team_rank_rev = team_rank_rev ; + m_league_size = team.league_size(); + m_league_rank = ( team.league_size() * pool_league_rank ) / pool_league_size ; + m_league_end = ( team.league_size() * (pool_league_rank+1) ) / pool_league_size ; + + set_team_shared(); + } + } + } + + bool valid() const + { return m_league_rank < m_league_end ; } + + void next() + { + if ( ++m_league_rank < m_league_end ) { + team_barrier(); + set_team_shared(); + } + } +}; +} /* namespace Impl */ +} /* namespace Kokkos */ + +//---------------------------------------------------------------------------- +//---------------------------------------------------------------------------- + +namespace Kokkos { + +inline int Threads::in_parallel() +{ return Impl::ThreadsExec::in_parallel(); } + +inline int Threads::is_initialized() +{ return Impl::ThreadsExec::is_initialized(); } + +inline void Threads::initialize( + unsigned threads_count , + unsigned use_numa_count , + unsigned use_cores_per_numa , + bool allow_asynchronous_threadpool ) +{ + Impl::ThreadsExec::initialize( threads_count , use_numa_count , use_cores_per_numa , allow_asynchronous_threadpool ); +} + +inline void Threads::finalize() +{ + Impl::ThreadsExec::finalize(); +} + +inline void Threads::print_configuration( std::ostream & s , const bool detail ) +{ + Impl::ThreadsExec::print_configuration( s , detail ); +} + +inline bool Threads::sleep() +{ return Impl::ThreadsExec::sleep() ; } + +inline bool Threads::wake() +{ return Impl::ThreadsExec::wake() ; } + +inline void Threads::fence() +{ Impl::ThreadsExec::fence() ; } + +} /* namespace Kokkos */ + +//---------------------------------------------------------------------------- +//---------------------------------------------------------------------------- + +namespace Kokkos { + +template< class Arg0 , class Arg1 > +class TeamPolicy< Arg0 , Arg1 , Kokkos::Threads > +{ +private: + + int m_league_size ; + int m_team_size ; + int m_team_alloc ; + + inline + void init( const int league_size_request + , const int team_size_request ) + { + const int pool_size = execution_space::thread_pool_size(0); + const int team_max = execution_space::thread_pool_size(1); + const int team_grain = execution_space::thread_pool_size(2); + + m_league_size = league_size_request ; + + m_team_size = team_size_request < team_max ? + team_size_request : team_max ; + + // Round team size up to a multiple of 'team_gain' + const int team_size_grain = team_grain * ( ( m_team_size + team_grain - 1 ) / team_grain ); + const int team_count = pool_size / team_size_grain ; + + // Constraint : pool_size = m_team_alloc * team_count + m_team_alloc = pool_size / team_count ; + } + + +public: + + //! Tag this class as a kokkos execution policy + typedef TeamPolicy execution_policy ; + typedef Kokkos::Threads execution_space ; + + typedef typename + Impl::if_c< ! Impl::is_same< Kokkos::Threads , Arg0 >::value , Arg0 , Arg1 >::type + work_tag ; + + //---------------------------------------- + + template< class FunctorType > + inline static + int team_size_max( const FunctorType & ) + { return execution_space::thread_pool_size(1); } + + template< class FunctorType > + static int team_size_recommended( const FunctorType & ) + { return execution_space::thread_pool_size(2); } + + //---------------------------------------- + + inline int team_size() const { return m_team_size ; } + inline int team_alloc() const { return m_team_alloc ; } + inline int league_size() const { return m_league_size ; } + + /** \brief Specify league size, request team size */ + TeamPolicy( execution_space & , int league_size_request , int team_size_request , int vector_length_request = 1 ) + : m_league_size(0) + , m_team_size(0) + , m_team_alloc(0) + { init(league_size_request,team_size_request); (void) vector_length_request; } + + TeamPolicy( int league_size_request , int team_size_request , int vector_length_request = 1 ) + : m_league_size(0) + , m_team_size(0) + , m_team_alloc(0) + { init(league_size_request,team_size_request); (void) vector_length_request; } + + typedef Impl::ThreadsExecTeamMember member_type ; + + friend class Impl::ThreadsExecTeamMember ; +}; + + +} /* namespace Kokkos */ + + +#ifdef KOKKOS_HAVE_CXX11 + +namespace Kokkos { + +template +KOKKOS_INLINE_FUNCTION +Impl::TeamThreadLoopBoundariesStruct + TeamThreadLoop(const Impl::ThreadsExecTeamMember& thread, const iType& count) { + return Impl::TeamThreadLoopBoundariesStruct(thread,count); +} + +template +KOKKOS_INLINE_FUNCTION +Impl::ThreadVectorLoopBoundariesStruct + ThreadVectorLoop(const Impl::ThreadsExecTeamMember& thread, const iType& count) { + return Impl::ThreadVectorLoopBoundariesStruct(thread,count); +} + + +KOKKOS_INLINE_FUNCTION +Impl::ThreadSingleStruct PerTeam(const Impl::ThreadsExecTeamMember& thread) { + return Impl::ThreadSingleStruct(thread); +} + +KOKKOS_INLINE_FUNCTION +Impl::VectorSingleStruct PerThread(const Impl::ThreadsExecTeamMember& thread) { + return Impl::VectorSingleStruct(thread); +} +} // namespace Kokkos + +namespace Kokkos { + + /** \brief Inter-thread parallel_for. Executes lambda(iType i) for each i=0..N-1. + * + * The range i=0..N-1 is mapped to all threads of the the calling thread team. + * This functionality requires C++11 support.*/ +template +KOKKOS_INLINE_FUNCTION +void parallel_for(const Impl::TeamThreadLoopBoundariesStruct& loop_boundaries, const Lambda& lambda) { + for( iType i = loop_boundaries.start; i < loop_boundaries.end; i+=loop_boundaries.increment) + lambda(i); +} + +/** \brief Inter-thread vector parallel_reduce. Executes lambda(iType i, ValueType & val) for each i=0..N-1. + * + * The range i=0..N-1 is mapped to all threads of the the calling thread team and a summation of + * val is performed and put into result. This functionality requires C++11 support.*/ +template< typename iType, class Lambda, typename ValueType > +KOKKOS_INLINE_FUNCTION +void parallel_reduce(const Impl::TeamThreadLoopBoundariesStruct& loop_boundaries, + const Lambda & lambda, ValueType& result) { + + result = ValueType(); + + for( iType i = loop_boundaries.start; i < loop_boundaries.end; i+=loop_boundaries.increment) { + ValueType tmp = ValueType(); + lambda(i,tmp); + result+=tmp; + } + + result = loop_boundaries.thread.team_reduce(result,Impl::JoinAdd()); +} + +/** \brief Intra-thread vector parallel_reduce. Executes lambda(iType i, ValueType & val) for each i=0..N-1. + * + * The range i=0..N-1 is mapped to all vector lanes of the the calling thread and a reduction of + * val is performed using JoinType(ValueType& val, const ValueType& update) and put into init_result. + * The input value of init_result is used as initializer for temporary variables of ValueType. Therefore + * the input value should be the neutral element with respect to the join operation (e.g. '0 for +-' or + * '1 for *'). This functionality requires C++11 support.*/ +template< typename iType, class Lambda, typename ValueType, class JoinType > +KOKKOS_INLINE_FUNCTION +void parallel_reduce(const Impl::TeamThreadLoopBoundariesStruct& loop_boundaries, + const Lambda & lambda, const JoinType& join, ValueType& init_result) { + + ValueType result = init_result; + + for( iType i = loop_boundaries.start; i < loop_boundaries.end; i+=loop_boundaries.increment) { + ValueType tmp = ValueType(); + lambda(i,tmp); + join(result,tmp); + } + + init_result = loop_boundaries.thread.team_reduce(result,Impl::JoinLambdaAdapter(join)); +} + +} //namespace Kokkos + + +namespace Kokkos { +/** \brief Intra-thread vector parallel_for. Executes lambda(iType i) for each i=0..N-1. + * + * The range i=0..N-1 is mapped to all vector lanes of the the calling thread. + * This functionality requires C++11 support.*/ +template +KOKKOS_INLINE_FUNCTION +void parallel_for(const Impl::ThreadVectorLoopBoundariesStruct& + loop_boundaries, const Lambda& lambda) { + #ifdef KOKKOS_HAVE_PRAGMA_IVDEP + #pragma ivdep + #endif + for( iType i = loop_boundaries.start; i < loop_boundaries.end; i+=loop_boundaries.increment) + lambda(i); +} + +/** \brief Intra-thread vector parallel_reduce. Executes lambda(iType i, ValueType & val) for each i=0..N-1. + * + * The range i=0..N-1 is mapped to all vector lanes of the the calling thread and a summation of + * val is performed and put into result. This functionality requires C++11 support.*/ +template< typename iType, class Lambda, typename ValueType > +KOKKOS_INLINE_FUNCTION +void parallel_reduce(const Impl::ThreadVectorLoopBoundariesStruct& + loop_boundaries, const Lambda & lambda, ValueType& result) { + result = ValueType(); +#ifdef KOKKOS_HAVE_PRAGMA_IVDEP +#pragma ivdep +#endif + for( iType i = loop_boundaries.start; i < loop_boundaries.end; i+=loop_boundaries.increment) { + ValueType tmp = ValueType(); + lambda(i,tmp); + result+=tmp; + } +} + +/** \brief Intra-thread vector parallel_reduce. Executes lambda(iType i, ValueType & val) for each i=0..N-1. + * + * The range i=0..N-1 is mapped to all vector lanes of the the calling thread and a reduction of + * val is performed using JoinType(ValueType& val, const ValueType& update) and put into init_result. + * The input value of init_result is used as initializer for temporary variables of ValueType. Therefore + * the input value should be the neutral element with respect to the join operation (e.g. '0 for +-' or + * '1 for *'). This functionality requires C++11 support.*/ +template< typename iType, class Lambda, typename ValueType, class JoinType > +KOKKOS_INLINE_FUNCTION +void parallel_reduce(const Impl::ThreadVectorLoopBoundariesStruct& + loop_boundaries, const Lambda & lambda, const JoinType& join, ValueType& init_result) { + + ValueType result = init_result; +#ifdef KOKKOS_HAVE_PRAGMA_IVDEP +#pragma ivdep +#endif + for( iType i = loop_boundaries.start; i < loop_boundaries.end; i+=loop_boundaries.increment) { + ValueType tmp = ValueType(); + lambda(i,tmp); + join(result,tmp); + } + init_result = result; +} + +/** \brief Intra-thread vector parallel exclusive prefix sum. Executes lambda(iType i, ValueType & val, bool final) + * for each i=0..N-1. + * + * The range i=0..N-1 is mapped to all vector lanes in the thread and a scan operation is performed. + * Depending on the target execution space the operator might be called twice: once with final=false + * and once with final=true. When final==true val contains the prefix sum value. The contribution of this + * "i" needs to be added to val no matter whether final==true or not. In a serial execution + * (i.e. team_size==1) the operator is only called once with final==true. Scan_val will be set + * to the final sum value over all vector lanes. + * This functionality requires C++11 support.*/ +template< typename iType, class FunctorType > +KOKKOS_INLINE_FUNCTION +void parallel_scan(const Impl::ThreadVectorLoopBoundariesStruct& + loop_boundaries, const FunctorType & lambda) { + + typedef Kokkos::Impl::FunctorValueTraits< FunctorType , void > ValueTraits ; + typedef typename ValueTraits::value_type value_type ; + + value_type scan_val = value_type(); + +#ifdef KOKKOS_HAVE_PRAGMA_IVDEP +#pragma ivdep +#endif + for( iType i = loop_boundaries.start; i < loop_boundaries.end; i+=loop_boundaries.increment) { + lambda(i,scan_val,true); + } +} + +} // namespace Kokkos + +namespace Kokkos { + +template +KOKKOS_INLINE_FUNCTION +void single(const Impl::VectorSingleStruct& single_struct, const FunctorType& lambda) { + lambda(); +} + +template +KOKKOS_INLINE_FUNCTION +void single(const Impl::ThreadSingleStruct& single_struct, const FunctorType& lambda) { + if(single_struct.team_member.team_rank()==0) lambda(); +} + +template +KOKKOS_INLINE_FUNCTION +void single(const Impl::VectorSingleStruct& single_struct, const FunctorType& lambda, ValueType& val) { + lambda(val); +} + +template +KOKKOS_INLINE_FUNCTION +void single(const Impl::ThreadSingleStruct& single_struct, const FunctorType& lambda, ValueType& val) { + if(single_struct.team_member.team_rank()==0) { + lambda(val); + } + single_struct.team_member.team_broadcast(val,0); +} +} +#endif // KOKKOS_HAVE_CXX11 + +//---------------------------------------------------------------------------- +//---------------------------------------------------------------------------- + +#endif /* #define KOKKOS_THREADSEXEC_HPP */ + diff --git a/lib/kokkos/core/src/Threads/Kokkos_ThreadsExec_base.cpp b/lib/kokkos/core/src/Threads/Kokkos_ThreadsExec_base.cpp new file mode 100755 index 0000000000..1c875328c7 --- /dev/null +++ b/lib/kokkos/core/src/Threads/Kokkos_ThreadsExec_base.cpp @@ -0,0 +1,254 @@ +/* +//@HEADER +// ************************************************************************ +// +// Kokkos: Manycore Performance-Portable Multidimensional Arrays +// Copyright (2012) Sandia Corporation +// +// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +// the U.S. Government retains certain rights in this software. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// 1. Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// 3. Neither the name of the Corporation nor the names of the +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Questions? Contact H. Carter Edwards (hcedwar@sandia.gov) +// +// ************************************************************************ +//@HEADER +*/ + +#include + +//---------------------------------------------------------------------------- +//---------------------------------------------------------------------------- + +#if defined( KOKKOS_HAVE_PTHREAD ) + +/* Standard 'C' Linux libraries */ + +#include +#include +#include + +/* Standard C++ libaries */ + +#include +#include +#include +#include + +#include + +//---------------------------------------------------------------------------- + +namespace Kokkos { +namespace Impl { +namespace { + +pthread_mutex_t host_internal_pthread_mutex = PTHREAD_MUTEX_INITIALIZER ; + +// Pthreads compatible driver. +// Recovery from an exception would require constant intra-thread health +// verification; which would negatively impact runtime. As such simply +// abort the process. + +void * internal_pthread_driver( void * ) +{ + try { + ThreadsExec::driver(); + } + catch( const std::exception & x ) { + std::cerr << "Exception thrown from worker thread: " << x.what() << std::endl ; + std::cerr.flush(); + std::abort(); + } + catch( ... ) { + std::cerr << "Exception thrown from worker thread" << std::endl ; + std::cerr.flush(); + std::abort(); + } + return NULL ; +} + +} // namespace + +//---------------------------------------------------------------------------- +// Spawn a thread + +bool ThreadsExec::spawn() +{ + bool result = false ; + + pthread_attr_t attr ; + + if ( 0 == pthread_attr_init( & attr ) || + 0 == pthread_attr_setscope( & attr, PTHREAD_SCOPE_SYSTEM ) || + 0 == pthread_attr_setdetachstate( & attr, PTHREAD_CREATE_DETACHED ) ) { + + pthread_t pt ; + + result = 0 == pthread_create( & pt, & attr, internal_pthread_driver, 0 ); + } + + pthread_attr_destroy( & attr ); + + return result ; +} + +//---------------------------------------------------------------------------- + +bool ThreadsExec::is_process() +{ + static const pthread_t master_pid = pthread_self(); + + return pthread_equal( master_pid , pthread_self() ); +} + +void ThreadsExec::global_lock() +{ + pthread_mutex_lock( & host_internal_pthread_mutex ); +} + +void ThreadsExec::global_unlock() +{ + pthread_mutex_unlock( & host_internal_pthread_mutex ); +} + +//---------------------------------------------------------------------------- + +void ThreadsExec::wait_yield( volatile int & flag , const int value ) +{ + while ( value == flag ) { sched_yield(); } +} + +} // namespace Impl +} // namespace Kokkos + +/* end #if defined( KOKKOS_HAVE_PTHREAD ) */ +//---------------------------------------------------------------------------- +//---------------------------------------------------------------------------- + +#elif defined( KOKKOS_HAVE_WINTHREAD ) + +/* Windows libraries */ +#include +#include + +/* Standard C++ libaries */ + +#include +#include +#include +#include + +#include + +//---------------------------------------------------------------------------- +// Driver for each created pthread + +namespace Kokkos { +namespace Impl { +namespace { + +unsigned WINAPI internal_winthread_driver( void * arg ) +{ + ThreadsExec::driver(); + + return 0 ; +} + +class ThreadLockWindows { +private: + CRITICAL_SECTION m_handle ; + + ~ThreadLockWindows() + { DeleteCriticalSection( & m_handle ); } + + ThreadLockWindows(); + { InitializeCriticalSection( & m_handle ); } + + ThreadLockWindows( const ThreadLockWindows & ); + ThreadLockWindows & operator = ( const ThreadLockWindows & ); + +public: + + static ThreadLockWindows & singleton(); + + void lock() + { EnterCriticalSection( & m_handle ); } + + void unlock() + { LeaveCriticalSection( & m_handle ); } +}; + +ThreadLockWindows & ThreadLockWindows::singleton() +{ static ThreadLockWindows self ; return self ; } + +} // namespace <> +} // namespace Kokkos +} // namespace Impl + +//---------------------------------------------------------------------------- +//---------------------------------------------------------------------------- + +namespace Kokkos { +namespace Impl { + +// Spawn this thread + +bool ThreadsExec::spawn() +{ + unsigned Win32ThreadID = 0 ; + + HANDLE handle = + _beginthreadex(0,0,internal_winthread_driver,0,0, & Win32ThreadID ); + + return ! handle ; +} + +bool ThreadsExec::is_process() { return true ; } + +void ThreadsExec::global_lock() +{ ThreadLockWindows::singleton().lock(); } + +void ThreadsExec::global_unlock() +{ ThreadLockWindows::singleton().unlock(); } + +void ThreadsExec::wait_yield( volatile int & flag , const int value ) {} +{ + while ( value == flag ) { Sleep(0); } +} + +} // namespace Impl +} // namespace Kokkos + +#endif /* end #elif defined( KOKKOS_HAVE_WINTHREAD ) */ +//---------------------------------------------------------------------------- +//---------------------------------------------------------------------------- + + + diff --git a/lib/kokkos/core/src/Threads/Kokkos_Threads_Parallel.hpp b/lib/kokkos/core/src/Threads/Kokkos_Threads_Parallel.hpp new file mode 100755 index 0000000000..4bb1b25f8f --- /dev/null +++ b/lib/kokkos/core/src/Threads/Kokkos_Threads_Parallel.hpp @@ -0,0 +1,427 @@ +/* +//@HEADER +// ************************************************************************ +// +// Kokkos: Manycore Performance-Portable Multidimensional Arrays +// Copyright (2012) Sandia Corporation +// +// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +// the U.S. Government retains certain rights in this software. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// 1. Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// 3. Neither the name of the Corporation nor the names of the +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Questions? Contact H. Carter Edwards (hcedwar@sandia.gov) +// +// ************************************************************************ +//@HEADER +*/ + +#ifndef KOKKOS_THREADS_PARALLEL_HPP +#define KOKKOS_THREADS_PARALLEL_HPP + +#include + +#include + +#include +#include + +//---------------------------------------------------------------------------- + +namespace Kokkos { +namespace Impl { + +//---------------------------------------------------------------------------- +//---------------------------------------------------------------------------- + +template< class FunctorType , class Arg0 , class Arg1 , class Arg2 > +class ParallelFor< FunctorType , Kokkos::RangePolicy< Arg0 , Arg1 , Arg2 , Kokkos::Threads > > +{ +private: + + typedef Kokkos::RangePolicy< Arg0 , Arg1 , Arg2 , Kokkos::Threads > Policy ; + + const FunctorType m_func ; + const Policy m_policy ; + + template< class PType > + KOKKOS_FORCEINLINE_FUNCTION static + void driver( typename Impl::enable_if< + ( Impl::is_same< typename PType::work_tag , void >::value ) + , const FunctorType & >::type functor + , const PType & range ) + { + const typename PType::member_type e = range.end(); + for ( typename PType::member_type i = range.begin() ; i < e ; ++i ) { + functor( i ); + } + } + + template< class PType > + KOKKOS_FORCEINLINE_FUNCTION static + void driver( typename Impl::enable_if< + ( ! Impl::is_same< typename PType::work_tag , void >::value ) + , const FunctorType & >::type functor + , const PType & range ) + { + const typename PType::member_type e = range.end(); + for ( typename PType::member_type i = range.begin() ; i < e ; ++i ) { + functor( typename PType::work_tag() , i ); + } + } + + static void execute( ThreadsExec & exec , const void * arg ) + { + const ParallelFor & self = * ((const ParallelFor *) arg ); + + driver( self.m_func , typename Policy::WorkRange( self.m_policy , exec.pool_rank() , exec.pool_size() ) ); + + exec.fan_in(); + } + +public: + + ParallelFor( const FunctorType & functor + , const Policy & policy ) + : m_func( functor ) + , m_policy( policy ) + { + ThreadsExec::start( & ParallelFor::execute , this ); + + ThreadsExec::fence(); + } +}; + +template< class FunctorType , class Arg0 , class Arg1 > +class ParallelFor< FunctorType , Kokkos::TeamPolicy< Arg0 , Arg1 , Kokkos::Threads > > +{ +private: + + typedef TeamPolicy< Arg0 , Arg1 , Kokkos::Threads > Policy ; + + const FunctorType m_func ; + const Policy m_policy ; + const int m_shared ; + + template< class TagType > + KOKKOS_FORCEINLINE_FUNCTION + void driver( typename Impl::enable_if< Impl::is_same< TagType , void >::value , + const typename Policy::member_type & >::type member ) const + { m_func( member ); } + + template< class TagType > + KOKKOS_FORCEINLINE_FUNCTION + void driver( typename Impl::enable_if< ! Impl::is_same< TagType , void >::value , + const typename Policy::member_type & >::type member ) const + { m_func( TagType() , member ); } + + static void execute( ThreadsExec & exec , const void * arg ) + { + const ParallelFor & self = * ((const ParallelFor *) arg ); + + typename Policy::member_type member( exec , self.m_policy , self.m_shared ); + + for ( ; member.valid() ; member.next() ) { + self.ParallelFor::template driver< typename Policy::work_tag >( member ); + } + + exec.fan_in(); + } + +public: + + ParallelFor( const FunctorType & functor + , const Policy & policy ) + : m_func( functor ) + , m_policy( policy ) + , m_shared( FunctorTeamShmemSize< FunctorType >::value( functor , policy.team_size() ) ) + { + ThreadsExec::resize_scratch( 0 , Policy::member_type::team_reduce_size() + m_shared ); + + ThreadsExec::start( & ParallelFor::execute , this ); + + ThreadsExec::fence(); + } +}; + + + +//---------------------------------------------------------------------------- +//---------------------------------------------------------------------------- + +template< class FunctorType , class Arg0 , class Arg1 , class Arg2 > +class ParallelReduce< FunctorType , Kokkos::RangePolicy< Arg0 , Arg1 , Arg2 , Kokkos::Threads > > +{ +private: + + typedef Kokkos::RangePolicy< Arg0 , Arg1 , Arg2 , Kokkos::Threads > Policy ; + typedef typename Policy::work_tag work_tag ; + typedef Kokkos::Impl::FunctorValueTraits< FunctorType , work_tag > ValueTraits ; + typedef Kokkos::Impl::FunctorValueInit< FunctorType , work_tag > ValueInit ; + + typedef typename ValueTraits::pointer_type pointer_type ; + typedef typename ValueTraits::reference_type reference_type ; + + const FunctorType m_func ; + const Policy m_policy ; + + template< class PType > + KOKKOS_FORCEINLINE_FUNCTION static + void driver( typename Impl::enable_if< + ( Impl::is_same< typename PType::work_tag , void >::value ) + , const FunctorType & >::type functor + , reference_type update + , const PType & range ) + { + const typename PType::member_type e = range.end(); + for ( typename PType::member_type i = range.begin() ; i < e ; ++i ) { + functor( i , update ); + } + } + + template< class PType > + KOKKOS_FORCEINLINE_FUNCTION static + void driver( typename Impl::enable_if< + ( ! Impl::is_same< typename PType::work_tag , void >::value ) + , const FunctorType & >::type functor + , reference_type update + , const PType & range ) + { + const typename PType::member_type e = range.end(); + for ( typename PType::member_type i = range.begin() ; i < e ; ++i ) { + functor( typename PType::work_tag() , i , update ); + } + } + + static void execute( ThreadsExec & exec , const void * arg ) + { + const ParallelReduce & self = * ((const ParallelReduce *) arg ); + + driver( self.m_func + , ValueInit::init( self.m_func , exec.reduce_memory() ) + , typename Policy::WorkRange( self.m_policy , exec.pool_rank() , exec.pool_size() ) + ); + + exec.template fan_in_reduce< FunctorType , work_tag >( self.m_func ); + } + +public: + + template< class HostViewType > + ParallelReduce( const FunctorType & functor , + const Policy & policy , + const HostViewType & result_view ) + : m_func( functor ) + , m_policy( policy ) + { + ThreadsExec::resize_scratch( ValueTraits::value_size( m_func ) , 0 ); + + ThreadsExec::start( & ParallelReduce::execute , this ); + + const pointer_type data = (pointer_type) ThreadsExec::root_reduce_scratch(); + + ThreadsExec::fence(); + + if ( result_view.ptr_on_device() ) { + const unsigned n = ValueTraits::value_count( m_func ); + for ( unsigned i = 0 ; i < n ; ++i ) { result_view.ptr_on_device()[i] = data[i]; } + } + } +}; + +//---------------------------------------------------------------------------- + +template< class FunctorType , class Arg0 , class Arg1 > +class ParallelReduce< FunctorType , Kokkos::TeamPolicy< Arg0 , Arg1 , Kokkos::Threads > > +{ +private: + + typedef TeamPolicy< Arg0 , Arg1 , Kokkos::Threads > Policy ; + typedef typename Policy::work_tag work_tag ; + typedef Kokkos::Impl::FunctorValueTraits< FunctorType , work_tag > ValueTraits ; + typedef Kokkos::Impl::FunctorValueInit< FunctorType , work_tag > ValueInit ; + + typedef typename ValueTraits::pointer_type pointer_type ; + typedef typename ValueTraits::reference_type reference_type ; + + const FunctorType m_func ; + const Policy m_policy ; + const int m_shared ; + + template< class TagType > + KOKKOS_FORCEINLINE_FUNCTION + void driver( typename Impl::enable_if< Impl::is_same< TagType , void >::value , + const typename Policy::member_type & >::type member + , reference_type update ) const + { m_func( member , update ); } + + template< class TagType > + KOKKOS_FORCEINLINE_FUNCTION + void driver( typename Impl::enable_if< ! Impl::is_same< TagType , void >::value , + const typename Policy::member_type & >::type member + , reference_type update ) const + { m_func( TagType() , member , update ); } + + static void execute( ThreadsExec & exec , const void * arg ) + { + const ParallelReduce & self = * ((const ParallelReduce *) arg ); + + // Initialize thread-local value + reference_type update = ValueInit::init( self.m_func , exec.reduce_memory() ); + + typename Policy::member_type member( exec , self.m_policy , self.m_shared ); + for ( ; member.valid() ; member.next() ) { + self.ParallelReduce::template driver< work_tag >( member , update ); + } + + exec.template fan_in_reduce< FunctorType , work_tag >( self.m_func ); + } + +public: + + ParallelReduce( const FunctorType & functor + , const Policy & policy ) + : m_func( functor ) + , m_policy( policy ) + , m_shared( FunctorTeamShmemSize< FunctorType >::value( functor , policy.team_size() ) ) + { + ThreadsExec::resize_scratch( ValueTraits::value_size( m_func ) , Policy::member_type::team_reduce_size() + m_shared ); + + ThreadsExec::start( & ParallelReduce::execute , this ); + + ThreadsExec::fence(); + } + + template< class ViewType > + ParallelReduce( const FunctorType & functor + , const Policy & policy + , const ViewType & result ) + : m_func( functor ) + , m_policy( policy ) + , m_shared( FunctorTeamShmemSize< FunctorType >::value( functor , policy.team_size() ) ) + { + ThreadsExec::resize_scratch( ValueTraits::value_size( m_func ) , Policy::member_type::team_reduce_size() + m_shared ); + + ThreadsExec::start( & ParallelReduce::execute , this ); + + const pointer_type data = (pointer_type) ThreadsExec::root_reduce_scratch(); + + ThreadsExec::fence(); + + const unsigned n = ValueTraits::value_count( m_func ); + for ( unsigned i = 0 ; i < n ; ++i ) { result.ptr_on_device()[i] = data[i]; } + } +}; + +//---------------------------------------------------------------------------- +//---------------------------------------------------------------------------- + +template< class FunctorType , class Arg0 , class Arg1 , class Arg2 > +class ParallelScan< FunctorType , Kokkos::RangePolicy< Arg0 , Arg1 , Arg2 , Kokkos::Threads > > +{ +private: + + typedef Kokkos::RangePolicy< Arg0 , Arg1 , Arg2 , Kokkos::Threads > Policy ; + typedef typename Policy::work_tag work_tag ; + typedef Kokkos::Impl::FunctorValueTraits< FunctorType , work_tag > ValueTraits ; + typedef Kokkos::Impl::FunctorValueInit< FunctorType , work_tag > ValueInit ; + + typedef typename ValueTraits::pointer_type pointer_type ; + typedef typename ValueTraits::reference_type reference_type ; + + const FunctorType m_func ; + const Policy m_policy ; + + template< class PType > + KOKKOS_FORCEINLINE_FUNCTION static + void driver( typename Impl::enable_if< + ( Impl::is_same< typename PType::work_tag , void >::value ) + , const FunctorType & >::type functor + , reference_type update + , const bool final + , const PType & range ) + { + const typename PType::member_type e = range.end(); + for ( typename PType::member_type i = range.begin() ; i < e ; ++i ) { + functor( i , update , final ); + } + } + + template< class PType > + KOKKOS_FORCEINLINE_FUNCTION static + void driver( typename Impl::enable_if< + ( ! Impl::is_same< typename PType::work_tag , void >::value ) + , const FunctorType & >::type functor + , reference_type update + , const bool final + , const PType & range ) + { + const typename PType::member_type e = range.end(); + for ( typename PType::member_type i = range.begin() ; i < e ; ++i ) { + functor( typename PType::work_tag() , i , update , final ); + } + } + + static void execute( ThreadsExec & exec , const void * arg ) + { + const ParallelScan & self = * ((const ParallelScan *) arg ); + + const typename Policy::WorkRange range( self.m_policy , exec.pool_rank() , exec.pool_size() ); + + reference_type update = ValueInit::init( self.m_func , exec.reduce_memory() ); + + driver( self.m_func , update , false , range ); + + // exec.scan_large( self.m_func ); + exec.template scan_small( self.m_func ); + + driver( self.m_func , update , true , range ); + + exec.fan_in(); + } + +public: + + ParallelScan( const FunctorType & functor , const Policy & policy ) + : m_func( functor ) + , m_policy( policy ) + { + ThreadsExec::resize_scratch( 2 * ValueTraits::value_size( m_func ) , 0 ); + ThreadsExec::start( & ParallelScan::execute , this ); + ThreadsExec::fence(); + } +}; + +} // namespace Impl +} // namespace Kokkos + +//---------------------------------------------------------------------------- +//---------------------------------------------------------------------------- + +#endif /* #define KOKKOS_THREADS_PARALLEL_HPP */ + diff --git a/lib/kokkos/core/src/build.cuda.mac b/lib/kokkos/core/src/build.cuda.mac new file mode 100755 index 0000000000..8c94550b7b --- /dev/null +++ b/lib/kokkos/core/src/build.cuda.mac @@ -0,0 +1,28 @@ +#!/bin/bash + +touch KokkosCore_config.h + +#flags="-I../ -I./ -I../../../TPL -c -O3 -arch=sm_30 -Xcompiler -fPIC -DKOKKOS_HAVE_CUDA -DKOKKOS_HAVE_PTHREAD --compiler-bindir=/Users/mhoemme/pkg/gcc-4.7.2/bin" +flags="-I../ -I./ -I../../../TPL -c -O3 -arch=sm_30 -Xcompiler -fPIC -DKOKKOS_HAVE_CUDA -DKOKKOS_HAVE_PTHREAD" +CC=nvcc +cd Cuda +rm *.o +$CC $flags Kokkos_Cuda_Impl.cu +$CC $flags Kokkos_CudaSpace.cu +cd .. +cd impl +rm *.o +$CC $flags Kokkos_hwloc.cpp +$CC $flags Kokkos_MemoryTracking.cpp +$CC $flags Kokkos_Shape.cpp +$CC $flags Kokkos_Error.cpp +$CC $flags Kokkos_HostSpace.cpp +$CC $flags Kokkos_Serial.cpp +cd .. +cd Threads +rm *.o +$CC $flags Kokkos_ThreadsExec.cpp +$CC $flags Kokkos_ThreadsExec_base.cpp +cd .. +$CC -arch=sm_35 -lib -o libkokkoscore-cuda.a Cuda/*.o impl/*.o Threads/*.o + diff --git a/lib/kokkos/core/src/build_common.sh b/lib/kokkos/core/src/build_common.sh new file mode 100755 index 0000000000..e029e51235 --- /dev/null +++ b/lib/kokkos/core/src/build_common.sh @@ -0,0 +1,281 @@ +#!/bin/bash + +#----------------------------------------------------------------------------- +# Shared portion of build script for the base Kokkos functionality +# Simple build script with options +#----------------------------------------------------------------------------- +if [ ! -d "${KOKKOS}" \ + -o ! -d "${KOKKOS}/src" \ + -o ! -d "${KOKKOS}/src/impl" \ + -o ! -d "${KOKKOS}/src/Cuda" \ + -o ! -d "${KOKKOS}/src/OpenMP" \ + -o ! -d "${KOKKOS}/src/Threads" \ + ] ; +then +echo "Must set KOKKOS to the kokkos/core directory" +exit -1 +fi + +#----------------------------------------------------------------------------- + +INC_PATH="-I${KOKKOS}/src" +INC_PATH="${INC_PATH} -I${KOKKOS}/../TPL" + +#----------------------------------------------------------------------------- + +while [ -n "${1}" ] ; do + +ARG="${1}" +shift 1 + +case ${ARG} in +#----------- OPTIONS ----------- +OPT | opt | O3 | -O3 ) OPTFLAGS="${OPTFLAGS} -O3" ;; +#------------------------------- +DBG | dbg | g | -g ) KOKKOS_EXPRESSION_CHECK=1 ;; +#------------------------------- +HWLOC | hwloc ) KOKKOS_HAVE_HWLOC=${1} ; shift 1 ;; +#------------------------------- +MPI | mpi ) + KOKKOS_HAVE_MPI=${1} ; shift 1 + CXX="${KOKKOS_HAVE_MPI}/bin/mpicxx" + LINK="${KOKKOS_HAVE_MPI}/bin/mpicxx" + INC_PATH="${INC_PATH} -I${KOKKOS_HAVE_MPI}/include" + ;; +#------------------------------- +OMP | omp | OpenMP ) + KOKKOS_HAVE_OPENMP=1 + ;; +#------------------------------- +CUDA | Cuda | cuda ) + # CUDA_ARCH options: 20 30 35 + CUDA_ARCH=${1} ; shift 1 + # + # -x cu : process all files through the Cuda compiler as Cuda code. + # -lib -o : produce library + # + NVCC="nvcc -gencode arch=compute_${CUDA_ARCH},code=sm_${CUDA_ARCH}" + NVCC="${NVCC} -maxrregcount=64" + NVCC="${NVCC} -Xcompiler -Wall,-ansi" + NVCC="${NVCC} -lib -o libCuda.a -x cu" + + NVCC_SOURCES="${NVCC_SOURCES} ${KOKKOS}/src/Cuda/*.cu" + LIB="${LIB} libCuda.a -L/usr/local/cuda/lib64 -lcudart -lcusparse" + ;;#------------------------------- +CUDA_OSX | Cuda_OSX | cuda_osx ) + # CUDA_ARCH options: 20 30 35 + CUDA_ARCH=${1} ; shift 1 + # + # -x cu : process all files through the Cuda compiler as Cuda code. + # -lib -o : produce library + # + NVCC="nvcc -gencode arch=compute_${CUDA_ARCH},code=sm_${CUDA_ARCH}" + NVCC="${NVCC} -maxrregcount=64" + NVCC="${NVCC} -Xcompiler -Wall,-ansi -Xcompiler -m64" + NVCC="${NVCC} -lib -o libCuda.a -x cu" + + NVCC_SOURCES="${NVCC_SOURCES} ${KOKKOS}/src/Cuda/*.cu" + LIB="${LIB} libCuda.a -Xlinker -rpath -Xlinker /Developer/NVIDIA/CUDA-5.5/lib -L /Developer/NVIDIA/CUDA-5.5/lib -lcudart -lcusparse" + ;; +#------------------------------- +GNU | gnu | g++ ) + # Turn on lots of warnings and ansi compliance. + # The Trilinos build system requires '-pedantic' + # + CXX="g++ -Wall -Wextra -ansi -pedantic" + LINK="g++" + CXX="${CXX} -rdynamic -DENABLE_TRACEBACK" + LIB="${LIB} -ldl" + ;; +#------------------------------- +GNU_OSX | gnu_osx | g++_osx ) + # Turn on lots of warnings and ansi compliance. + # The Trilinos build system requires '-pedantic' + # + CXX="g++ -Wall -Wextra -ansi -pedantic -m64" + LINK="g++" + CXX="${CXX} -DENABLE_TRACEBACK" + LIB="${LIB} -ldl" + ;; +#------------------------------- +INTEL | intel | icc | icpc ) + # -xW = use SSE and SSE2 instructions + CXX="icpc -Wall" + LINK="icpc" + LIB="${LIB} -lstdc++" + ;; +#------------------------------- +MPIINTEL | mpiintel | mpiicc | mpiicpc ) + # -xW = use SSE and SSE2 instructions + CXX="mpiicpc -Wall" + LINK="mpiicpc" + LIB="${LIB} -lstdc++" + KOKKOS_HAVE_MPI=1 +;; +#------------------------------- +MIC | mic ) + CXX="icpc -mmic -ansi-alias -Wall" + LINK="icpc -mmic" + CXX="${CXX} -mGLOB_default_function_attrs=knc_stream_store_controls=2" + # CXX="${CXX} -vec-report6" + # CXX="${CXX} -guide-vec" + LIB="${LIB} -lstdc++" + COMPILE_MIC="on" + ;; +#------------------------------- +MPIMIC | mpimic ) + CXX="mpiicpc -mmic -ansi-alias -Wall" + LINK="mpiicpc -mmic" + KOKKOS_HAVE_MPI=1 + CXX="${CXX} -mGLOB_default_function_attrs=knc_stream_store_controls=2" + # CXX="${CXX} -vec-report6" + # CXX="${CXX} -guide-vec" + LIB="${LIB} -lstdc++" + COMPILE_MIC="on" + ;; +#------------------------------- +curie ) + CXX="CC" + LINK="CC" + INC_PATH="${INC_PATH} -I/opt/cray/mpt/default/gni/mpich2-cray/74" + KOKKOS_HAVE_MPI=1 + ;; +#------------------------------- +MKL | mkl ) + HAVE_MKL=${1} ; shift 1 ; + CXX_FLAGS="${CXX_FLAGS} -DKOKKOS_USE_MKL -I${HAVE_MKL}/include/" + ARCH="intel64" + if [ -n "${COMPILE_MIC}" ] ; + then + ARCH="mic" + fi + LIB="${LIB} -L${HAVE_MKL}/lib/${ARCH}/ -lmkl_intel_lp64 -lmkl_intel_thread -lmkl_core" + NVCC_FLAGS="${NVCC_FLAGS} -DKOKKOS_USE_MKL" +;; +#------------------------------- +CUSPARSE | cusparse ) + CXX_FLAGS="${CXX_FLAGS} -DKOKKOS_USE_CUSPARSE" + NVCC_FLAGS="${NVCC_FLAGS} -DKOKKOS_USE_CUSPARSE" + LIB="${LIB} -lcusparse" +;; +#------------------------------- +AVX | avx ) + CXX_FLAGS="${CXX_FLAGS} -mavx" +;; +#------------------------------- +*) echo 'unknown option: ' ${ARG} ; exit -1 ;; +esac +done + +#----------------------------------------------------------------------------- + +if [ -z "${CXX}" ] ; +then + echo "No C++ compiler selected" + exit -1 +fi + +if [ -n "${KOKKOS_HAVE_OPENMP}" ] +then +CXX="${CXX} -fopenmp" +CXX_SOURCES="${CXX_SOURCES} ${KOKKOS}/src/OpenMP/*.cpp" +fi + +#----------------------------------------------------------------------------- +# Option for PTHREAD or WINTHREAD eventually + +KOKKOS_HAVE_PTHREAD=1 + +if [ -n "${KOKKOS_HAVE_PTHREAD}" ] ; +then + LIB="${LIB} -lpthread" +fi + +#----------------------------------------------------------------------------- +# Option for enabling the Serial device + +KOKKOS_HAVE_SERIAL=1 + +#----------------------------------------------------------------------------- +# Attach options to compile lines + +CXX="${CXX} ${OPTFLAGS}" + +if [ -n "${NVCC}" ] ; +then + NVCC="${NVCC} ${OPTFLAGS}" +fi + +#----------------------------------------------------------------------------- + +CXX_SOURCES="${CXX_SOURCES} ${KOKKOS}/src/impl/*.cpp" +CXX_SOURCES="${CXX_SOURCES} ${KOKKOS}/src/Threads/*.cpp" + +#----------------------------------------------------------------------------- +# + +if [ -n "${KOKKOS_HAVE_HWLOC}" ] ; +then + + if [ ! -d ${KOKKOS_HAVE_HWLOC} ] ; + then + echo "${KOKKOS_HAVE_HWLOC} does not exist" + exit 1 + fi + + echo "LD_LIBRARY_PATH must include ${KOKKOS_HAVE_HWLOC}/lib" + + LIB="${LIB} -L${KOKKOS_HAVE_HWLOC}/lib -lhwloc" + INC_PATH="${INC_PATH} -I${KOKKOS_HAVE_HWLOC}/include" +fi + +#----------------------------------------------------------------------------- + +INC_PATH="${INC_PATH} -I." + +CONFIG="KokkosCore_config.h" + +rm -f ${CONFIG} + +echo "#ifndef KOKKOS_CORE_CONFIG_H" >> ${CONFIG} +echo "#define KOKKOS_CORE_CONFIG_H" >> ${CONFIG} + +if [ -n "${KOKKOS_HAVE_MPI}" ] ; +then + echo "#define KOKKOS_HAVE_MPI" >> ${CONFIG} +fi + +if [ -n "${NVCC}" ] ; +then + echo "#define KOKKOS_HAVE_CUDA" >> ${CONFIG} +fi + +if [ -n "${KOKKOS_HAVE_PTHREAD}" ] ; +then + echo "#define KOKKOS_HAVE_PTHREAD" >> ${CONFIG} +fi + +if [ -n "${KOKKOS_HAVE_SERIAL}" ] ; +then + echo "#define KOKKOS_HAVE_SERIAL" >> ${CONFIG} +fi + +if [ -n "${KOKKOS_HAVE_HWLOC}" ] ; +then + echo "#define KOKKOS_HAVE_HWLOC" >> ${CONFIG} +fi + +if [ -n "${KOKKOS_HAVE_OPENMP}" ] ; +then + echo "#define KOKKOS_HAVE_OPENMP" >> ${CONFIG} +fi + +if [ -n "${KOKKOS_EXPRESSION_CHECK}" ] ; +then + echo "#define KOKKOS_EXPRESSION_CHECK" >> ${CONFIG} +fi + +echo "#endif" >> ${CONFIG} + +#----------------------------------------------------------------------------- + diff --git a/lib/kokkos/core/src/impl/Kokkos_AnalyzeShape.hpp b/lib/kokkos/core/src/impl/Kokkos_AnalyzeShape.hpp new file mode 100755 index 0000000000..b2330248cc --- /dev/null +++ b/lib/kokkos/core/src/impl/Kokkos_AnalyzeShape.hpp @@ -0,0 +1,260 @@ +/* +//@HEADER +// ************************************************************************ +// +// Kokkos: Manycore Performance-Portable Multidimensional Arrays +// Copyright (2012) Sandia Corporation +// +// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +// the U.S. Government retains certain rights in this software. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// 1. Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// 3. Neither the name of the Corporation nor the names of the +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Questions? Contact H. Carter Edwards (hcedwar@sandia.gov) +// +// ************************************************************************ +//@HEADER +*/ + +#ifndef KOKKOS_ANALYZESHAPE_HPP +#define KOKKOS_ANALYZESHAPE_HPP + +#include + +//---------------------------------------------------------------------------- +//---------------------------------------------------------------------------- + +namespace Kokkos { +namespace Impl { + +//---------------------------------------------------------------------------- + +/** \brief Analyze the array shape defined by a Kokkos::View data type. + * + * It is presumed that the data type can be mapped down to a multidimensional + * array of an intrinsic scalar numerical type (double, float, int, ... ). + * The 'value_type' of an array may be an embedded aggregate type such + * as a fixed length array 'Array'. + * In this case the 'array_intrinsic_type' represents the + * underlying array of intrinsic scalar numerical type. + * + * The embedded aggregate type must have an AnalyzeShape specialization + * to map it down to a shape and intrinsic scalar numerical type. + */ +template< class T > +struct AnalyzeShape : public Shape< sizeof(T) , 0 > +{ + typedef void specialize ; + + typedef Shape< sizeof(T), 0 > shape ; + + typedef T array_intrinsic_type ; + typedef T value_type ; + typedef T type ; + + typedef const T const_array_intrinsic_type ; + typedef const T const_value_type ; + typedef const T const_type ; + + typedef T non_const_array_intrinsic_type ; + typedef T non_const_value_type ; + typedef T non_const_type ; +}; + +template<> +struct AnalyzeShape : public Shape< 0 , 0 > +{ + typedef void specialize ; + + typedef Shape< 0 , 0 > shape ; + + typedef void array_intrinsic_type ; + typedef void value_type ; + typedef void type ; + typedef const void const_array_intrinsic_type ; + typedef const void const_value_type ; + typedef const void const_type ; + typedef void non_const_array_intrinsic_type ; + typedef void non_const_value_type ; + typedef void non_const_type ; +}; + +template< class T > +struct AnalyzeShape< const T > : public AnalyzeShape::shape +{ +private: + typedef AnalyzeShape nested ; +public: + + typedef typename nested::specialize specialize ; + + typedef typename nested::shape shape ; + + typedef typename nested::const_array_intrinsic_type array_intrinsic_type ; + typedef typename nested::const_value_type value_type ; + typedef typename nested::const_type type ; + + typedef typename nested::const_array_intrinsic_type const_array_intrinsic_type ; + typedef typename nested::const_value_type const_value_type ; + typedef typename nested::const_type const_type ; + + typedef typename nested::non_const_array_intrinsic_type non_const_array_intrinsic_type ; + typedef typename nested::non_const_value_type non_const_value_type ; + typedef typename nested::non_const_type non_const_type ; +}; + +template< class T > +struct AnalyzeShape< T * > + : public ShapeInsert< typename AnalyzeShape::shape , 0 >::type +{ +private: + typedef AnalyzeShape nested ; +public: + + typedef typename nested::specialize specialize ; + + typedef typename ShapeInsert< typename nested::shape , 0 >::type shape ; + + typedef typename nested::array_intrinsic_type * array_intrinsic_type ; + typedef typename nested::value_type value_type ; + typedef typename nested::type * type ; + + typedef typename nested::const_array_intrinsic_type * const_array_intrinsic_type ; + typedef typename nested::const_value_type const_value_type ; + typedef typename nested::const_type * const_type ; + + typedef typename nested::non_const_array_intrinsic_type * non_const_array_intrinsic_type ; + typedef typename nested::non_const_value_type non_const_value_type ; + typedef typename nested::non_const_type * non_const_type ; +}; + +template< class T > +struct AnalyzeShape< T[] > + : public ShapeInsert< typename AnalyzeShape::shape , 0 >::type +{ +private: + typedef AnalyzeShape nested ; +public: + + typedef typename nested::specialize specialize ; + + typedef typename ShapeInsert< typename nested::shape , 0 >::type shape ; + + typedef typename nested::array_intrinsic_type array_intrinsic_type [] ; + typedef typename nested::value_type value_type ; + typedef typename nested::type type [] ; + + typedef typename nested::const_array_intrinsic_type const_array_intrinsic_type [] ; + typedef typename nested::const_value_type const_value_type ; + typedef typename nested::const_type const_type [] ; + + typedef typename nested::non_const_array_intrinsic_type non_const_array_intrinsic_type [] ; + typedef typename nested::non_const_value_type non_const_value_type ; + typedef typename nested::non_const_type non_const_type [] ; +}; + +template< class T > +struct AnalyzeShape< const T[] > + : public ShapeInsert< typename AnalyzeShape< const T >::shape , 0 >::type +{ +private: + typedef AnalyzeShape< const T > nested ; +public: + + typedef typename nested::specialize specialize ; + + typedef typename ShapeInsert< typename nested::shape , 0 >::type shape ; + + typedef typename nested::array_intrinsic_type array_intrinsic_type [] ; + typedef typename nested::value_type value_type ; + typedef typename nested::type type [] ; + + typedef typename nested::const_array_intrinsic_type const_array_intrinsic_type [] ; + typedef typename nested::const_value_type const_value_type ; + typedef typename nested::const_type const_type [] ; + + typedef typename nested::non_const_array_intrinsic_type non_const_array_intrinsic_type [] ; + typedef typename nested::non_const_value_type non_const_value_type ; + typedef typename nested::non_const_type non_const_type [] ; +}; + +template< class T , unsigned N > +struct AnalyzeShape< T[N] > + : public ShapeInsert< typename AnalyzeShape::shape , N >::type +{ +private: + typedef AnalyzeShape nested ; +public: + + typedef typename nested::specialize specialize ; + + typedef typename ShapeInsert< typename nested::shape , N >::type shape ; + + typedef typename nested::array_intrinsic_type array_intrinsic_type [N] ; + typedef typename nested::value_type value_type ; + typedef typename nested::type type [N] ; + + typedef typename nested::const_array_intrinsic_type const_array_intrinsic_type [N] ; + typedef typename nested::const_value_type const_value_type ; + typedef typename nested::const_type const_type [N] ; + + typedef typename nested::non_const_array_intrinsic_type non_const_array_intrinsic_type [N] ; + typedef typename nested::non_const_value_type non_const_value_type ; + typedef typename nested::non_const_type non_const_type [N] ; +}; + +template< class T , unsigned N > +struct AnalyzeShape< const T[N] > + : public ShapeInsert< typename AnalyzeShape< const T >::shape , N >::type +{ +private: + typedef AnalyzeShape< const T > nested ; +public: + + typedef typename nested::specialize specialize ; + + typedef typename ShapeInsert< typename nested::shape , N >::type shape ; + + typedef typename nested::array_intrinsic_type array_intrinsic_type [N] ; + typedef typename nested::value_type value_type ; + typedef typename nested::type type [N] ; + + typedef typename nested::const_array_intrinsic_type const_array_intrinsic_type [N] ; + typedef typename nested::const_value_type const_value_type ; + typedef typename nested::const_type const_type [N] ; + + typedef typename nested::non_const_array_intrinsic_type non_const_array_intrinsic_type [N] ; + typedef typename nested::non_const_value_type non_const_value_type ; + typedef typename nested::non_const_type non_const_type [N] ; +}; + +} // namespace Impl +} // namespace Kokkos + +#endif /* #ifndef KOKKOS_ANALYZESHAPE_HPP */ + diff --git a/lib/kokkos/core/src/impl/Kokkos_Atomic_Assembly_X86.hpp b/lib/kokkos/core/src/impl/Kokkos_Atomic_Assembly_X86.hpp new file mode 100755 index 0000000000..b1ce1bd44e --- /dev/null +++ b/lib/kokkos/core/src/impl/Kokkos_Atomic_Assembly_X86.hpp @@ -0,0 +1,176 @@ +/* +//@HEADER +// ************************************************************************ +// +// Kokkos +// Manycore Performance-Portable Multidimensional Arrays +// +// Copyright (2012) Sandia Corporation +// +// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +// the U.S. Government retains certain rights in this software. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// 1. Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// 3. Neither the name of the Corporation nor the names of the +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Questions? Contact H. Carter Edwards (hcedwar@sandia.gov) +// +// ************************************************************************ +//@HEADER +*/ +#if defined( KOKKOS_ATOMIC_HPP ) && ! defined( KOKKOS_ATOMIC_ASSEMBLY_X86_HPP ) +#define KOKKOS_ATOMIC_ASSEMBLY_X86_HPP +namespace Kokkos { + +#ifndef __CUDA_ARCH__ +template<> +KOKKOS_INLINE_FUNCTION +void atomic_increment(volatile char* a) { + __asm__ __volatile__( + "lock incb %0" + : /* no output registers */ + : "m" (a[0]) + : "memory" + ); +} + +template<> +KOKKOS_INLINE_FUNCTION +void atomic_increment(volatile short* a) { + __asm__ __volatile__( + "lock incw %0" + : /* no output registers */ + : "m" (a[0]) + : "memory" + ); +} + +template<> +KOKKOS_INLINE_FUNCTION +void atomic_increment(volatile int* a) { + __asm__ __volatile__( + "lock incl %0" + : /* no output registers */ + : "m" (a[0]) + : "memory" + ); +} + +template<> +KOKKOS_INLINE_FUNCTION +void atomic_increment(volatile long long int* a) { + __asm__ __volatile__( + "lock incq %0" + : /* no output registers */ + : "m" (a[0]) + : "memory" + ); +} + +template<> +KOKKOS_INLINE_FUNCTION +void atomic_decrement(volatile char* a) { + __asm__ __volatile__( + "lock decb %0" + : /* no output registers */ + : "m" (a[0]) + : "memory" + ); +} + +template<> +KOKKOS_INLINE_FUNCTION +void atomic_decrement(volatile short* a) { + __asm__ __volatile__( + "lock decw %0" + : /* no output registers */ + : "m" (a[0]) + : "memory" + ); +} + +template<> +KOKKOS_INLINE_FUNCTION +void atomic_decrement(volatile int* a) { + __asm__ __volatile__( + "lock decl %0" + : /* no output registers */ + : "m" (a[0]) + : "memory" + ); +} + +template<> +KOKKOS_INLINE_FUNCTION +void atomic_decrement(volatile long long int* a) { + __asm__ __volatile__( + "lock decq %0" + : /* no output registers */ + : "m" (a[0]) + : "memory" + ); +} +#endif + +namespace Impl { + struct cas128_t + { + uint64_t lower; + uint64_t upper; + KOKKOS_INLINE_FUNCTION + bool operator != (const cas128_t& a) const { + return (lower != a.lower) || upper!=a.upper; + } + } + __attribute__ (( __aligned__( 16 ) )); + + + + + inline cas128_t cas128( volatile cas128_t * ptr, cas128_t cmp, cas128_t swap ) + { + bool swapped; + __asm__ __volatile__ + ( + "lock cmpxchg16b %1\n\t" + "setz %0" + : "=q" ( swapped ) + , "+m" ( *ptr ) + , "+d" ( cmp.upper ) + , "+a" ( cmp.lower ) + : "c" ( swap.upper ) + , "b" ( swap.lower ) + : "cc" + ); + (void) swapped; + return cmp; + } + +} +} + +#endif diff --git a/lib/kokkos/core/src/impl/Kokkos_Atomic_Compare_Exchange_Strong.hpp b/lib/kokkos/core/src/impl/Kokkos_Atomic_Compare_Exchange_Strong.hpp new file mode 100755 index 0000000000..a1c35d9f9d --- /dev/null +++ b/lib/kokkos/core/src/impl/Kokkos_Atomic_Compare_Exchange_Strong.hpp @@ -0,0 +1,231 @@ +/* +//@HEADER +// ************************************************************************ +// +// Kokkos: Manycore Performance-Portable Multidimensional Arrays +// Copyright (2012) Sandia Corporation +// +// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +// the U.S. Government retains certain rights in this software. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// 1. Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// 3. Neither the name of the Corporation nor the names of the +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Questions? Contact H. Carter Edwards (hcedwar@sandia.gov) +// +// ************************************************************************ +//@HEADER +*/ + +#if defined( KOKKOS_ATOMIC_HPP ) && ! defined( KOKKOS_ATOMIC_COMPARE_EXCHANGE_STRONG_HPP ) +#define KOKKOS_ATOMIC_COMPARE_EXCHANGE_STRONG_HPP + +namespace Kokkos { + +//---------------------------------------------------------------------------- +// Cuda native CAS supports int, unsigned int, and unsigned long long int (non-standard type). +// Must cast-away 'volatile' for the CAS call. + +#if defined( KOKKOS_ATOMICS_USE_CUDA ) + +__inline__ __device__ +int atomic_compare_exchange( volatile int * const dest, const int compare, const int val) +{ return atomicCAS((int*)dest,compare,val); } + +__inline__ __device__ +unsigned int atomic_compare_exchange( volatile unsigned int * const dest, const unsigned int compare, const unsigned int val) +{ return atomicCAS((unsigned int*)dest,compare,val); } + +__inline__ __device__ +unsigned long long int atomic_compare_exchange( volatile unsigned long long int * const dest , + const unsigned long long int compare , + const unsigned long long int val ) +{ return atomicCAS((unsigned long long int*)dest,compare,val); } + +template < typename T > +__inline__ __device__ +T atomic_compare_exchange( volatile T * const dest , const T & compare , + typename Kokkos::Impl::enable_if< sizeof(T) == sizeof(int) , const T & >::type val ) +{ + const int tmp = atomicCAS( (int*) dest , *((int*)&compare) , *((int*)&val) ); + return *((T*)&tmp); +} + +template < typename T > +__inline__ __device__ +T atomic_compare_exchange( volatile T * const dest , const T & compare , + typename Kokkos::Impl::enable_if< sizeof(T) != sizeof(int) && + sizeof(T) == sizeof(unsigned long long int) , const T & >::type val ) +{ + typedef unsigned long long int type ; + const type tmp = atomicCAS( (type*) dest , *((type*)&compare) , *((type*)&val) ); + return *((T*)&tmp); +} + +template < typename T > +__inline__ __device__ +T atomic_compare_exchange( volatile T * const dest , const T & compare , + typename Kokkos::Impl::enable_if< sizeof(T) != sizeof(int) && + sizeof(T) != sizeof(unsigned long long int) && + sizeof(T) == sizeof(Impl::cas128_t) , const T & >::type val ) +{ + Kokkos::abort("Error: calling atomic_compare_exchange with 128bit type is not supported on CUDA execution space."); + return T(); +} + +//---------------------------------------------------------------------------- +// GCC native CAS supports int, long, unsigned int, unsigned long. +// Intel native CAS support int and long with the same interface as GCC. + +#elif defined(KOKKOS_ATOMICS_USE_GCC) || defined(KOKKOS_ATOMICS_USE_INTEL) + +KOKKOS_INLINE_FUNCTION +int atomic_compare_exchange( volatile int * const dest, const int compare, const int val) +{ return __sync_val_compare_and_swap(dest,compare,val); } + +KOKKOS_INLINE_FUNCTION +long atomic_compare_exchange( volatile long * const dest, const long compare, const long val ) +{ return __sync_val_compare_and_swap(dest,compare,val); } + +#if defined( KOKKOS_ATOMICS_USE_GCC ) + +// GCC supports unsigned + +KOKKOS_INLINE_FUNCTION +unsigned int atomic_compare_exchange( volatile unsigned int * const dest, const unsigned int compare, const unsigned int val ) +{ return __sync_val_compare_and_swap(dest,compare,val); } + +KOKKOS_INLINE_FUNCTION +unsigned long atomic_compare_exchange( volatile unsigned long * const dest , + const unsigned long compare , + const unsigned long val ) +{ return __sync_val_compare_and_swap(dest,compare,val); } + +#endif + +template < typename T > +KOKKOS_INLINE_FUNCTION +T atomic_compare_exchange( volatile T * const dest, const T & compare, + typename Kokkos::Impl::enable_if< sizeof(T) == sizeof(int) , const T & >::type val ) +{ +#ifdef KOKKOS_HAVE_CXX11 + union U { + int i ; + T t ; + KOKKOS_INLINE_FUNCTION U() {}; + } tmp ; +#else + union U { + int i ; + T t ; + } tmp ; +#endif + + tmp.i = __sync_val_compare_and_swap( (int*) dest , *((int*)&compare) , *((int*)&val) ); + return tmp.t ; +} + +template < typename T > +KOKKOS_INLINE_FUNCTION +T atomic_compare_exchange( volatile T * const dest, const T & compare, + typename Kokkos::Impl::enable_if< sizeof(T) != sizeof(int) && + sizeof(T) == sizeof(long) , const T & >::type val ) +{ +#ifdef KOKKOS_HAVE_CXX11 + union U { + long i ; + T t ; + KOKKOS_INLINE_FUNCTION U() {}; + } tmp ; +#else + union U { + long i ; + T t ; + } tmp ; +#endif + + tmp.i = __sync_val_compare_and_swap( (long*) dest , *((long*)&compare) , *((long*)&val) ); + return tmp.t ; +} + +template < typename T > +KOKKOS_INLINE_FUNCTION +T atomic_compare_exchange( volatile T * const dest, const T & compare, + typename Kokkos::Impl::enable_if< sizeof(T) != sizeof(int) && + sizeof(T) != sizeof(long) && + sizeof(T) == sizeof(Impl::cas128_t), const T & >::type val ) +{ +#ifdef KOKKOS_HAVE_CXX11 + union U { + Impl::cas128_t i ; + T t ; + KOKKOS_INLINE_FUNCTION U() {}; + } tmp ; +#else + union U { + Impl::cas128_t i ; + T t ; + } tmp ; +#endif + + tmp.i = Impl::cas128( (Impl::cas128_t*) dest , *((Impl::cas128_t*)&compare) , *((Impl::cas128_t*)&val) ); + return tmp.t ; +} +//---------------------------------------------------------------------------- + +#elif defined( KOKKOS_ATOMICS_USE_OMP31 ) + +template< typename T > +KOKKOS_INLINE_FUNCTION +T atomic_compare_exchange( volatile T * const dest, const T compare, const T val ) +{ + T retval; +#pragma omp critical + { + retval = dest[0]; + if ( retval == compare ) + dest[0] = val; + } + return retval; +} + +#endif + + +template +KOKKOS_INLINE_FUNCTION +bool atomic_compare_exchange_strong(volatile T* const dest, const T compare, const T val) +{ + return compare == atomic_compare_exchange(dest, compare, val); +} + +//---------------------------------------------------------------------------- + +} // namespace Kokkos + +#endif + diff --git a/lib/kokkos/core/src/impl/Kokkos_Atomic_Exchange.hpp b/lib/kokkos/core/src/impl/Kokkos_Atomic_Exchange.hpp new file mode 100755 index 0000000000..b39adf2a9b --- /dev/null +++ b/lib/kokkos/core/src/impl/Kokkos_Atomic_Exchange.hpp @@ -0,0 +1,305 @@ +/* +//@HEADER +// ************************************************************************ +// +// Kokkos: Manycore Performance-Portable Multidimensional Arrays +// Copyright (2012) Sandia Corporation +// +// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +// the U.S. Government retains certain rights in this software. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// 1. Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// 3. Neither the name of the Corporation nor the names of the +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Questions? Contact H. Carter Edwards (hcedwar@sandia.gov) +// +// ************************************************************************ +//@HEADER +*/ + +#if defined( KOKKOS_ATOMIC_HPP ) && ! defined( KOKKOS_ATOMIC_EXCHANGE_HPP ) +#define KOKKOS_ATOMIC_EXCHANGE_HPP + +namespace Kokkos { + +//---------------------------------------------------------------------------- + +#if defined( KOKKOS_ATOMICS_USE_CUDA ) + +__inline__ __device__ +int atomic_exchange( volatile int * const dest , const int val ) +{ + // return __iAtomicExch( (int*) dest , val ); + return atomicExch( (int*) dest , val ); +} + +__inline__ __device__ +unsigned int atomic_exchange( volatile unsigned int * const dest , const unsigned int val ) +{ + // return __uAtomicExch( (unsigned int*) dest , val ); + return atomicExch( (unsigned int*) dest , val ); +} + +__inline__ __device__ +unsigned long long int atomic_exchange( volatile unsigned long long int * const dest , const unsigned long long int val ) +{ + // return __ullAtomicExch( (unsigned long long*) dest , val ); + return atomicExch( (unsigned long long*) dest , val ); +} + +/** \brief Atomic exchange for any type with compatible size */ +template< typename T > +__inline__ __device__ +T atomic_exchange( + volatile T * const dest , + typename Kokkos::Impl::enable_if< sizeof(T) == sizeof(int) , const T & >::type val ) +{ + // int tmp = __ullAtomicExch( (int*) dest , *((int*)&val) ); + int tmp = atomicExch( ((int*)dest) , *((int*)&val) ); + return *((T*)&tmp); +} + +template< typename T > +__inline__ __device__ +T atomic_exchange( + volatile T * const dest , + typename Kokkos::Impl::enable_if< sizeof(T) != sizeof(int) && + sizeof(T) == sizeof(unsigned long long int) , const T & >::type val ) +{ + typedef unsigned long long int type ; + // type tmp = __ullAtomicExch( (type*) dest , *((type*)&val) ); + type tmp = atomicExch( ((type*)dest) , *((type*)&val) ); + return *((T*)&tmp); +} + +template< typename T > +__inline__ __device__ +T atomic_exchange( + volatile T * const dest , + typename Kokkos::Impl::enable_if< sizeof(T) != sizeof(int) && + sizeof(T) != sizeof(unsigned long long int) && + sizeof(T) == sizeof(Impl::cas128_t) , const T & >::type val ) +{ + Kokkos::abort("Error: calling atomic_exchange with 128bit type is not supported on CUDA execution space."); + return T(); +} + +/** \brief Atomic exchange for any type with compatible size */ +template< typename T > +__inline__ __device__ +void atomic_assign( + volatile T * const dest , + typename Kokkos::Impl::enable_if< sizeof(T) == sizeof(int) , const T & >::type val ) +{ + // (void) __ullAtomicExch( (int*) dest , *((int*)&val) ); + (void) atomicExch( ((int*)dest) , *((int*)&val) ); +} + +template< typename T > +__inline__ __device__ +void atomic_assign( + volatile T * const dest , + typename Kokkos::Impl::enable_if< sizeof(T) != sizeof(int) && + sizeof(T) == sizeof(unsigned long long int) , const T & >::type val ) +{ + typedef unsigned long long int type ; + // (void) __ullAtomicExch( (type*) dest , *((type*)&val) ); + (void) atomicExch( ((type*)dest) , *((type*)&val) ); +} + +template< typename T > +__inline__ __device__ +void atomic_assign( + volatile T * const dest , + typename Kokkos::Impl::enable_if< sizeof(T) != sizeof(int) && + sizeof(T) != sizeof(unsigned long long int) && + sizeof(T) == sizeof(Impl::cas128_t) , const T & >::type val ) +{ + Kokkos::abort("Error: calling atomic_assign with 128bit type is not supported on CUDA execution space."); +} + +//---------------------------------------------------------------------------- + +#elif defined(KOKKOS_ATOMICS_USE_GCC) || defined(KOKKOS_ATOMICS_USE_INTEL) + +template< typename T > +KOKKOS_INLINE_FUNCTION +T atomic_exchange( volatile T * const dest , + typename Kokkos::Impl::enable_if< sizeof(T) == sizeof(int) || sizeof(T) == sizeof(long) + , const T & >::type val ) +{ + typedef typename Kokkos::Impl::if_c< sizeof(T) == sizeof(int) , int , long >::type type ; + + const type v = *((type*)&val); // Extract to be sure the value doesn't change + + type assumed ; + +#ifdef KOKKOS_HAVE_CXX11 + union U { + T val_T ; + type val_type ; + KOKKOS_INLINE_FUNCTION U() {}; + } old ; +#else + union { T val_T ; type val_type ; } old ; +#endif + + old.val_T = *dest ; + + do { + assumed = old.val_type ; + old.val_type = __sync_val_compare_and_swap( (volatile type *) dest , assumed , v ); + } while ( assumed != old.val_type ); + + return old.val_T ; +} + +template< typename T > +KOKKOS_INLINE_FUNCTION +T atomic_exchange( volatile T * const dest , + typename Kokkos::Impl::enable_if< sizeof(T) == sizeof(Impl::cas128_t) + , const T & >::type val ) +{ +#ifdef KOKKOS_HAVE_CXX11 + union U { + Impl::cas128_t i ; + T t ; + KOKKOS_INLINE_FUNCTION U() {}; + } assume , oldval , newval ; +#else + union U { + Impl::cas128_t i ; + T t ; + } assume , oldval , newval ; +#endif + + oldval.t = *dest ; + newval.t = val; + + do { + assume.i = oldval.i ; + oldval.i = Impl::cas128( (volatile Impl::cas128_t*) dest , assume.i , newval.i ); + } while ( assume.i != oldval.i ); + + return oldval.t ; +} + +template< typename T > +KOKKOS_INLINE_FUNCTION +void atomic_assign( volatile T * const dest , + typename Kokkos::Impl::enable_if< sizeof(T) == sizeof(int) || sizeof(T) == sizeof(long) + , const T & >::type val ) +{ + typedef typename Kokkos::Impl::if_c< sizeof(T) == sizeof(int) , int , long >::type type ; + + const type v = *((type*)&val); // Extract to be sure the value doesn't change + + type assumed ; + +#ifdef KOKKOS_HAVE_CXX11 + union U { + T val_T ; + type val_type ; + KOKKOS_INLINE_FUNCTION U() {}; + } old ; +#else + union { T val_T ; type val_type ; } old ; +#endif + + old.val_T = *dest ; + + do { + assumed = old.val_type ; + old.val_type = __sync_val_compare_and_swap( (volatile type *) dest , assumed , v ); + } while ( assumed != old.val_type ); +} + +template< typename T > +KOKKOS_INLINE_FUNCTION +void atomic_assign( volatile T * const dest , + typename Kokkos::Impl::enable_if< sizeof(T) == sizeof(Impl::cas128_t) + , const T & >::type val ) +{ +#ifdef KOKKOS_HAVE_CXX11 + union U { + Impl::cas128_t i ; + T t ; + KOKKOS_INLINE_FUNCTION U() {}; + } assume , oldval , newval ; +#else + union U { + Impl::cas128_t i ; + T t ; + } assume , oldval , newval ; +#endif + + oldval.t = *dest ; + newval.t = val; + do { + assume.i = oldval.i ; + oldval.i = Impl::cas128( (volatile Impl::cas128_t*) dest , assume.i , newval.i); + } while ( assume.i != oldval.i ); +} +//---------------------------------------------------------------------------- + +#elif defined( KOKKOS_ATOMICS_USE_OMP31 ) + +template < typename T > +KOKKOS_INLINE_FUNCTION +T atomic_exchange( volatile T * const dest , const T val ) +{ + T retval; +//#pragma omp atomic capture + #pragma omp critical + { + retval = dest[0]; + dest[0] = val; + } + return retval; +} + +template < typename T > +KOKKOS_INLINE_FUNCTION +void atomic_assign( volatile T * const dest , const T val ) +{ +//#pragma omp atomic + #pragma omp critical + { + dest[0] = val; + } +} + +#endif + +//---------------------------------------------------------------------------- + +} // namespace Kokkos + +#endif + +//---------------------------------------------------------------------------- + diff --git a/lib/kokkos/core/src/impl/Kokkos_Atomic_Fetch_Add.hpp b/lib/kokkos/core/src/impl/Kokkos_Atomic_Fetch_Add.hpp new file mode 100755 index 0000000000..ce8b9a093d --- /dev/null +++ b/lib/kokkos/core/src/impl/Kokkos_Atomic_Fetch_Add.hpp @@ -0,0 +1,297 @@ +/* +//@HEADER +// ************************************************************************ +// +// Kokkos: Manycore Performance-Portable Multidimensional Arrays +// Copyright (2012) Sandia Corporation +// +// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +// the U.S. Government retains certain rights in this software. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// 1. Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// 3. Neither the name of the Corporation nor the names of the +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Questions? Contact H. Carter Edwards (hcedwar@sandia.gov) +// +// ************************************************************************ +//@HEADER +*/ + +#if defined( KOKKOS_ATOMIC_HPP ) && ! defined( KOKKOS_ATOMIC_FETCH_ADD_HPP ) +#define KOKKOS_ATOMIC_FETCH_ADD_HPP + +namespace Kokkos { + +//---------------------------------------------------------------------------- + +#if defined( KOKKOS_ATOMICS_USE_CUDA ) + +// Support for int, unsigned int, unsigned long long int, and float + +__inline__ __device__ +int atomic_fetch_add( volatile int * const dest , const int val ) +{ return atomicAdd((int*)dest,val); } + +__inline__ __device__ +unsigned int atomic_fetch_add( volatile unsigned int * const dest , const unsigned int val ) +{ return atomicAdd((unsigned int*)dest,val); } + +__inline__ __device__ +unsigned long long int atomic_fetch_add( volatile unsigned long long int * const dest , + const unsigned long long int val ) +{ return atomicAdd((unsigned long long int*)dest,val); } + +__inline__ __device__ +float atomic_fetch_add( volatile float * const dest , const float val ) +{ return atomicAdd((float*)dest,val); } + +template < typename T > +__inline__ __device__ +T atomic_fetch_add( volatile T * const dest , + typename Kokkos::Impl::enable_if< sizeof(T) == sizeof(int) , const T >::type val ) +{ +#ifdef KOKKOS_HAVE_CXX11 + union U { + int i ; + T t ; + KOKKOS_INLINE_FUNCTION U() {}; + } assume , oldval , newval ; +#else + union U { + int i ; + T t ; + } assume , oldval , newval ; +#endif + + oldval.t = *dest ; + + do { + assume.i = oldval.i ; + newval.t = assume.t + val ; + oldval.i = atomicCAS( (int*)dest , assume.i , newval.i ); + } while ( assumed.i != oldval.i ); + + return oldval.t ; +} + +template < typename T > +__inline__ __device__ +T atomic_fetch_add( volatile T * const dest , + typename Kokkos::Impl::enable_if< sizeof(T) != sizeof(int) && + sizeof(T) == sizeof(unsigned long long int) , const T >::type val ) +{ +#ifdef KOKKOS_HAVE_CXX11 + union U { + unsigned long long int i ; + T t ; + KOKKOS_INLINE_FUNCTION U() {}; + } assume , oldval , newval ; +#else + union U { + unsigned long long int i ; + T t ; + } assume , oldval , newval ; +#endif + + oldval.t = *dest ; + + do { + assume.i = oldval.i ; + newval.t = assume.t + val ; + oldval.i = atomicCAS( (unsigned long long int*)dest , assume.i , newval.i ); + } while ( assume.i != oldval.i ); + + return oldval.t ; +} + +template < typename T > +__inline__ __device__ +T atomic_fetch_add( volatile T * const dest , + typename Kokkos::Impl::enable_if< sizeof(T) != sizeof(int) && + sizeof(T) != sizeof(unsigned long long int) && + sizeof(T) == sizeof(Impl::cas128_t), const T >::type val ) +{ + Kokkos::abort("Error: calling atomic_fetch_add with 128bit type is not supported on CUDA execution space."); + return T(); +} + +//---------------------------------------------------------------------------- + +#elif defined(KOKKOS_ATOMICS_USE_GCC) || defined(KOKKOS_ATOMICS_USE_INTEL) + +KOKKOS_INLINE_FUNCTION +int atomic_fetch_add( volatile int * const dest , const int val ) +{ return __sync_fetch_and_add(dest,val); } + +KOKKOS_INLINE_FUNCTION +long int atomic_fetch_add( volatile long int * const dest , const long int val ) +{ return __sync_fetch_and_add(dest,val); } + +#if defined( KOKKOS_ATOMICS_USE_GCC ) + +KOKKOS_INLINE_FUNCTION +unsigned int atomic_fetch_add( volatile unsigned int * const dest , const unsigned int val ) +{ return __sync_fetch_and_add(dest,val); } + +KOKKOS_INLINE_FUNCTION +unsigned long int atomic_fetch_add( volatile unsigned long int * const dest , const unsigned long int val ) +{ return __sync_fetch_and_add(dest,val); } + +#endif + +template < typename T > +KOKKOS_INLINE_FUNCTION +T atomic_fetch_add( volatile T * const dest , + typename Kokkos::Impl::enable_if< sizeof(T) == sizeof(int) , const T >::type val ) +{ +#ifdef KOKKOS_HAVE_CXX11 + union U { + int i ; + T t ; + KOKKOS_INLINE_FUNCTION U() {}; + } assume , oldval , newval ; +#else + union U { + int i ; + T t ; + } assume , oldval , newval ; +#endif + + oldval.t = *dest ; + + do { + assume.i = oldval.i ; + newval.t = assume.t + val ; + oldval.i = __sync_val_compare_and_swap( (int*) dest , assume.i , newval.i ); + } while ( assume.i != oldval.i ); + + return oldval.t ; +} + +template < typename T > +KOKKOS_INLINE_FUNCTION +T atomic_fetch_add( volatile T * const dest , + typename Kokkos::Impl::enable_if< sizeof(T) != sizeof(int) && + sizeof(T) == sizeof(long) , const T >::type val ) +{ +#ifdef KOKKOS_HAVE_CXX11 + union U { + long i ; + T t ; + KOKKOS_INLINE_FUNCTION U() {}; + } assume , oldval , newval ; +#else + union U { + long i ; + T t ; + } assume , oldval , newval ; +#endif + + oldval.t = *dest ; + + do { + assume.i = oldval.i ; + newval.t = assume.t + val ; + oldval.i = __sync_val_compare_and_swap( (long*) dest , assume.i , newval.i ); + } while ( assume.i != oldval.i ); + + return oldval.t ; +} + +template < typename T > +KOKKOS_INLINE_FUNCTION +T atomic_fetch_add( volatile T * const dest , + typename Kokkos::Impl::enable_if< sizeof(T) != sizeof(int) && + sizeof(T) != sizeof(long) && + sizeof(T) == sizeof(Impl::cas128_t) , const T >::type val ) +{ +#ifdef KOKKOS_HAVE_CXX11 + union U { + Impl::cas128_t i ; + T t ; + KOKKOS_INLINE_FUNCTION U() {}; + } assume , oldval , newval ; +#else + union U { + Impl::cas128_t i ; + T t ; + } assume , oldval , newval ; +#endif + + oldval.t = *dest ; + + do { + assume.i = oldval.i ; + newval.t = assume.t + val ; + oldval.i = Impl::cas128( (volatile Impl::cas128_t*) dest , assume.i , newval.i ); + } while ( assume.i != oldval.i ); + + return oldval.t ; +} +//---------------------------------------------------------------------------- + +#elif defined( KOKKOS_ATOMICS_USE_OMP31 ) + +template< typename T > +T atomic_fetch_add( volatile T * const dest , const T val ) +{ + T retval; +#pragma omp atomic capture + { + retval = dest[0]; + dest[0] += val; + } + return retval; +} + +#endif + +//---------------------------------------------------------------------------- + +// Simpler version of atomic_fetch_add without the fetch +template +KOKKOS_INLINE_FUNCTION +void atomic_add(volatile T * const dest, const T src) { + atomic_fetch_add(dest,src); +} + +// Atomic increment +template +KOKKOS_INLINE_FUNCTION +void atomic_increment(volatile T* a) { + Kokkos::atomic_fetch_add(a,1); +} + +template +KOKKOS_INLINE_FUNCTION +void atomic_decrement(volatile T* a) { + Kokkos::atomic_fetch_add(a,-1); +} + +} +#endif + diff --git a/lib/kokkos/core/src/impl/Kokkos_Atomic_Fetch_And.hpp b/lib/kokkos/core/src/impl/Kokkos_Atomic_Fetch_And.hpp new file mode 100755 index 0000000000..9e62fd65d3 --- /dev/null +++ b/lib/kokkos/core/src/impl/Kokkos_Atomic_Fetch_And.hpp @@ -0,0 +1,125 @@ +/* +//@HEADER +// ************************************************************************ +// +// Kokkos: Manycore Performance-Portable Multidimensional Arrays +// Copyright (2012) Sandia Corporation +// +// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +// the U.S. Government retains certain rights in this software. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// 1. Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// 3. Neither the name of the Corporation nor the names of the +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Questions? Contact H. Carter Edwards (hcedwar@sandia.gov) +// +// ************************************************************************ +//@HEADER +*/ + +#if defined( KOKKOS_ATOMIC_HPP ) && ! defined( KOKKOS_ATOMIC_FETCH_AND_HPP ) +#define KOKKOS_ATOMIC_FETCH_AND_HPP + +namespace Kokkos { + +//---------------------------------------------------------------------------- + +#if defined( KOKKOS_ATOMICS_USE_CUDA ) + +// Support for int, unsigned int, unsigned long long int, and float + +__inline__ __device__ +int atomic_fetch_and( volatile int * const dest , const int val ) +{ return atomicAnd((int*)dest,val); } + +__inline__ __device__ +unsigned int atomic_fetch_and( volatile unsigned int * const dest , const unsigned int val ) +{ return atomicAnd((unsigned int*)dest,val); } + +#if defined( __CUDA_ARCH__ ) && ( 350 <= __CUDA_ARCH__ ) +__inline__ __device__ +unsigned long long int atomic_fetch_and( volatile unsigned long long int * const dest , + const unsigned long long int val ) +{ return atomicAnd((unsigned long long int*)dest,val); } +#endif + +//---------------------------------------------------------------------------- + +#elif defined(KOKKOS_ATOMICS_USE_GCC) || defined(KOKKOS_ATOMICS_USE_INTEL) + +KOKKOS_INLINE_FUNCTION +int atomic_fetch_and( volatile int * const dest , const int val ) +{ return __sync_fetch_and_and(dest,val); } + +KOKKOS_INLINE_FUNCTION +long int atomic_fetch_and( volatile long int * const dest , const long int val ) +{ return __sync_fetch_and_and(dest,val); } + +#if defined( KOKKOS_ATOMICS_USE_GCC ) + +KOKKOS_INLINE_FUNCTION +unsigned int atomic_fetch_and( volatile unsigned int * const dest , const unsigned int val ) +{ return __sync_fetch_and_and(dest,val); } + +KOKKOS_INLINE_FUNCTION +unsigned long int atomic_fetch_and( volatile unsigned long int * const dest , const unsigned long int val ) +{ return __sync_fetch_and_and(dest,val); } + +#endif + +//---------------------------------------------------------------------------- + +#elif defined( KOKKOS_ATOMICS_USE_OMP31 ) + +template< typename T > +T atomic_fetch_and( volatile T * const dest , const T val ) +{ + T retval; +#pragma omp atomic capture + { + retval = dest[0]; + dest[0] &= val; + } + return retval; +} + +#endif + +//---------------------------------------------------------------------------- + +// Simpler version of atomic_fetch_and without the fetch +template +KOKKOS_INLINE_FUNCTION +void atomic_and(volatile T * const dest, const T src) { + (void)atomic_fetch_and(dest,src); +} + +} + +#endif + + diff --git a/lib/kokkos/core/src/impl/Kokkos_Atomic_Fetch_Or.hpp b/lib/kokkos/core/src/impl/Kokkos_Atomic_Fetch_Or.hpp new file mode 100755 index 0000000000..22a4a7866d --- /dev/null +++ b/lib/kokkos/core/src/impl/Kokkos_Atomic_Fetch_Or.hpp @@ -0,0 +1,125 @@ +/* +//@HEADER +// ************************************************************************ +// +// Kokkos: Manycore Performance-Portable Multidimensional Arrays +// Copyright (2012) Sandia Corporation +// +// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +// the U.S. Government retains certain rights in this software. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// 1. Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// 3. Neither the name of the Corporation nor the names of the +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Questions? Contact H. Carter Edwards (hcedwar@sandia.gov) +// +// ************************************************************************ +//@HEADER +*/ + +#if defined( KOKKOS_ATOMIC_HPP ) && ! defined( KOKKOS_ATOMIC_FETCH_OR_HPP ) +#define KOKKOS_ATOMIC_FETCH_OR_HPP + +namespace Kokkos { + +//---------------------------------------------------------------------------- + +#if defined( KOKKOS_ATOMICS_USE_CUDA ) + +// Support for int, unsigned int, unsigned long long int, and float + +__inline__ __device__ +int atomic_fetch_or( volatile int * const dest , const int val ) +{ return atomicOr((int*)dest,val); } + +__inline__ __device__ +unsigned int atomic_fetch_or( volatile unsigned int * const dest , const unsigned int val ) +{ return atomicOr((unsigned int*)dest,val); } + +#if defined( __CUDA_ARCH__ ) && ( 350 <= __CUDA_ARCH__ ) +__inline__ __device__ +unsigned long long int atomic_fetch_or( volatile unsigned long long int * const dest , + const unsigned long long int val ) +{ return atomicOr((unsigned long long int*)dest,val); } +#endif + +//---------------------------------------------------------------------------- + +#elif defined(KOKKOS_ATOMICS_USE_GCC) || defined(KOKKOS_ATOMICS_USE_INTEL) + +KOKKOS_INLINE_FUNCTION +int atomic_fetch_or( volatile int * const dest , const int val ) +{ return __sync_fetch_and_or(dest,val); } + +KOKKOS_INLINE_FUNCTION +long int atomic_fetch_or( volatile long int * const dest , const long int val ) +{ return __sync_fetch_and_or(dest,val); } + +#if defined( KOKKOS_ATOMICS_USE_GCC ) + +KOKKOS_INLINE_FUNCTION +unsigned int atomic_fetch_or( volatile unsigned int * const dest , const unsigned int val ) +{ return __sync_fetch_and_or(dest,val); } + +KOKKOS_INLINE_FUNCTION +unsigned long int atomic_fetch_or( volatile unsigned long int * const dest , const unsigned long int val ) +{ return __sync_fetch_and_or(dest,val); } + +#endif + +//---------------------------------------------------------------------------- + +#elif defined( KOKKOS_ATOMICS_USE_OMP31 ) + +template< typename T > +T atomic_fetch_or( volatile T * const dest , const T val ) +{ + T retval; +#pragma omp atomic capture + { + retval = dest[0]; + dest[0] |= val; + } + return retval; +} + +#endif + +//---------------------------------------------------------------------------- + +// Simpler version of atomic_fetch_or without the fetch +template +KOKKOS_INLINE_FUNCTION +void atomic_or(volatile T * const dest, const T src) { + (void)atomic_fetch_or(dest,src); +} + +} + +#endif + + diff --git a/lib/kokkos/core/src/impl/Kokkos_Atomic_Generic.hpp b/lib/kokkos/core/src/impl/Kokkos_Atomic_Generic.hpp new file mode 100755 index 0000000000..1251428253 --- /dev/null +++ b/lib/kokkos/core/src/impl/Kokkos_Atomic_Generic.hpp @@ -0,0 +1,383 @@ +/* +//@HEADER +// ************************************************************************ +// +// Kokkos: Manycore Performance-Portable Multidimensional Arrays +// Copyright (2012) Sandia Corporation +// +// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +// the U.S. Government retains certain rights in this software. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// 1. Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// 3. Neither the name of the Corporation nor the names of the +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Questions? Contact H. Carter Edwards (hcedwar@sandia.gov) +// +// ************************************************************************ +//@HEADER +*/ +#if defined( KOKKOS_ATOMIC_HPP ) && ! defined( KOKKOS_ATOMIC_GENERIC_HPP ) +#define KOKKOS_ATOMIC_GENERIC_HPP +#include + +// Combination operands to be used in an Compare and Exchange based atomic operation +namespace Kokkos { +namespace Impl { + +template +struct AddOper { + KOKKOS_FORCEINLINE_FUNCTION + static Scalar1 apply(const Scalar1& val1, const Scalar2& val2) { + return val1+val2; + } +}; + +template +struct SubOper { + KOKKOS_FORCEINLINE_FUNCTION + static Scalar1 apply(const Scalar1& val1, const Scalar2& val2) { + return val1-val2; + } +}; + +template +struct MulOper { + KOKKOS_FORCEINLINE_FUNCTION + static Scalar1 apply(const Scalar1& val1, const Scalar2& val2) { + return val1*val2; + } +}; + +template +struct DivOper { + KOKKOS_FORCEINLINE_FUNCTION + static Scalar1 apply(const Scalar1& val1, const Scalar2& val2) { + return val1/val2; + } +}; + +template +struct ModOper { + KOKKOS_FORCEINLINE_FUNCTION + static Scalar1 apply(const Scalar1& val1, const Scalar2& val2) { + return val1%val2; + } +}; + +template +struct AndOper { + KOKKOS_FORCEINLINE_FUNCTION + static Scalar1 apply(const Scalar1& val1, const Scalar2& val2) { + return val1&val2; + } +}; + +template +struct OrOper { + KOKKOS_FORCEINLINE_FUNCTION + static Scalar1 apply(const Scalar1& val1, const Scalar2& val2) { + return val1|val2; + } +}; + +template +struct XorOper { + KOKKOS_FORCEINLINE_FUNCTION + static Scalar1 apply(const Scalar1& val1, const Scalar2& val2) { + return val1^val2; + } +}; + +template +struct LShiftOper { + KOKKOS_FORCEINLINE_FUNCTION + static Scalar1 apply(const Scalar1& val1, const Scalar2& val2) { + return val1< +struct RShiftOper { + KOKKOS_FORCEINLINE_FUNCTION + static Scalar1 apply(const Scalar1& val1, const Scalar2& val2) { + return val1>>val2; + } +}; + +template < class Oper, typename T > +KOKKOS_INLINE_FUNCTION +T atomic_fetch_oper( const Oper& op, volatile T * const dest , + typename ::Kokkos::Impl::enable_if< sizeof(T) != sizeof(int) && + sizeof(T) == sizeof(unsigned long long int) , const T >::type val ) +{ + union { unsigned long long int i ; T t ; } oldval , assume , newval ; + + oldval.t = *dest ; + + do { + assume.i = oldval.i ; + newval.t = Oper::apply(assume.t, val) ; + oldval.i = ::Kokkos::atomic_compare_exchange( (unsigned long long int*)dest , assume.i , newval.i ); + } while ( assume.i != oldval.i ); + + return oldval.t ; +} + +template < class Oper, typename T > +KOKKOS_INLINE_FUNCTION +T atomic_oper_fetch( const Oper& op, volatile T * const dest , + typename ::Kokkos::Impl::enable_if< sizeof(T) != sizeof(int) && + sizeof(T) == sizeof(unsigned long long int) , const T >::type val ) +{ + union { unsigned long long int i ; T t ; } oldval , assume , newval ; + + oldval.t = *dest ; + + do { + assume.i = oldval.i ; + newval.t = Oper::apply(assume.t, val) ; + oldval.i = ::Kokkos::atomic_compare_exchange( (unsigned long long int*)dest , assume.i , newval.i ); + } while ( assume.i != oldval.i ); + + return newval.t ; +} + +template < class Oper, typename T > +KOKKOS_INLINE_FUNCTION +T atomic_fetch_oper( const Oper& op, volatile T * const dest , + typename ::Kokkos::Impl::enable_if< sizeof(T) == sizeof(int) , const T >::type val ) +{ + union { int i ; T t ; } oldval , assume , newval ; + + oldval.t = *dest ; + + do { + assume.i = oldval.i ; + newval.t = Oper::apply(assume.t, val) ; + oldval.i = ::Kokkos::atomic_compare_exchange( (int*)dest , assume.i , newval.i ); + } while ( assume.i != oldval.i ); + + return oldval.t ; +} + +template < class Oper, typename T > +KOKKOS_INLINE_FUNCTION +T atomic_oper_fetch( const Oper& op, volatile T * const dest , + typename ::Kokkos::Impl::enable_if< sizeof(T) == sizeof(int), const T >::type val ) +{ + union { int i ; T t ; } oldval , assume , newval ; + + oldval.t = *dest ; + + do { + assume.i = oldval.i ; + newval.t = Oper::apply(assume.t, val) ; + oldval.i = ::Kokkos::atomic_compare_exchange( (int*)dest , assume.i , newval.i ); + } while ( assume.i != oldval.i ); + + return newval.t ; +} + +/*template < class Oper, typename T > +KOKKOS_INLINE_FUNCTION +T atomic_fetch_oper( const Oper& op, volatile T * const dest , + typename ::Kokkos::Impl::enable_if< sizeof(T) == sizeof(short) , const T >::type val ) +{ + union { short i ; T t ; } oldval , assume , newval ; + + oldval.t = *dest ; + + do { + assume.i = oldval.i ; + newval.t = Oper::apply(assume.t, val) ; + oldval.i = ::Kokkos::atomic_compare_exchange( (short*)dest , assume.i , newval.i ); + } while ( assume.i != oldval.i ); + + return oldval.t ; +} + +template < class Oper, typename T > +KOKKOS_INLINE_FUNCTION +T atomic_oper_fetch( const Oper& op, volatile T * const dest , + typename ::Kokkos::Impl::enable_if< sizeof(T) == sizeof(short), const T >::type val ) +{ + union { short i ; T t ; } oldval , assume , newval ; + + oldval.t = *dest ; + + do { + assume.i = oldval.i ; + newval.t = Oper::apply(assume.t, val) ; + oldval.i = ::Kokkos::atomic_compare_exchange( (short*)dest , assume.i , newval.i ); + } while ( assume.i != oldval.i ); + + return newval.t ; +} + +template < class Oper, typename T > +KOKKOS_INLINE_FUNCTION +T atomic_fetch_oper( const Oper& op, volatile T * const dest , + typename ::Kokkos::Impl::enable_if< sizeof(T) == sizeof(char) , const T >::type val ) +{ + union { char i ; T t ; } oldval , assume , newval ; + + oldval.t = *dest ; + + do { + assume.i = oldval.i ; + newval.t = Oper::apply(assume.t, val) ; + oldval.i = ::Kokkos::atomic_compare_exchange( (char*)dest , assume.i , newval.i ); + } while ( assume.i != oldval.i ); + + return oldval.t ; +} + +template < class Oper, typename T > +KOKKOS_INLINE_FUNCTION +T atomic_oper_fetch( const Oper& op, volatile T * const dest , + typename ::Kokkos::Impl::enable_if< sizeof(T) == sizeof(char), const T >::type val ) +{ + union { char i ; T t ; } oldval , assume , newval ; + + oldval.t = *dest ; + + do { + assume.i = oldval.i ; + newval.t = Oper::apply(assume.t, val) ; + oldval.i = ::Kokkos::atomic_compare_exchange( (char*)dest , assume.i , newval.i ); + } while ( assume.i != oldval.i ); + + return newval.t ; +}*/ + +} +} + +namespace Kokkos { + +// Fetch_Oper atomics: return value before operation +template < typename T > +KOKKOS_INLINE_FUNCTION +T atomic_fetch_mul(volatile T * const dest, const T val) { + return Impl::atomic_fetch_oper(Impl::MulOper(),dest,val); +} + +template < typename T > +KOKKOS_INLINE_FUNCTION +T atomic_fetch_div(volatile T * const dest, const T val) { + return Impl::atomic_fetch_oper(Impl::DivOper(),dest,val); +} + +template < typename T > +KOKKOS_INLINE_FUNCTION +T atomic_fetch_mod(volatile T * const dest, const T val) { + return Impl::atomic_fetch_oper(Impl::ModOper(),dest,val); +} + +template < typename T > +KOKKOS_INLINE_FUNCTION +T atomic_fetch_and(volatile T * const dest, const T val) { + return Impl::atomic_fetch_oper(Impl::AndOper(),dest,val); +} + +template < typename T > +KOKKOS_INLINE_FUNCTION +T atomic_fetch_or(volatile T * const dest, const T val) { + return Impl::atomic_fetch_oper(Impl::OrOper(),dest,val); +} + +template < typename T > +KOKKOS_INLINE_FUNCTION +T atomic_fetch_xor(volatile T * const dest, const T val) { + return Impl::atomic_fetch_oper(Impl::XorOper(),dest,val); +} + +template < typename T > +KOKKOS_INLINE_FUNCTION +T atomic_fetch_lshift(volatile T * const dest, const unsigned int val) { + return Impl::atomic_fetch_oper(Impl::LShiftOper(),dest,val); +} + +template < typename T > +KOKKOS_INLINE_FUNCTION +T atomic_fetch_rshift(volatile T * const dest, const unsigned int val) { + return Impl::atomic_fetch_oper(Impl::RShiftOper(),dest,val); +} + + +// Oper Fetch atomics: return value after operation +template < typename T > +KOKKOS_INLINE_FUNCTION +T atomic_mul_fetch(volatile T * const dest, const T val) { + return Impl::atomic_oper_fetch(Impl::MulOper(),dest,val); +} + +template < typename T > +KOKKOS_INLINE_FUNCTION +T atomic_div_fetch(volatile T * const dest, const T val) { + return Impl::atomic_oper_fetch(Impl::DivOper(),dest,val); +} + +template < typename T > +KOKKOS_INLINE_FUNCTION +T atomic_mod_fetch(volatile T * const dest, const T val) { + return Impl::atomic_oper_fetch(Impl::ModOper(),dest,val); +} + +template < typename T > +KOKKOS_INLINE_FUNCTION +T atomic_and_fetch(volatile T * const dest, const T val) { + return Impl::atomic_oper_fetch(Impl::AndOper(),dest,val); +} + +template < typename T > +KOKKOS_INLINE_FUNCTION +T atomic_or_fetch(volatile T * const dest, const T val) { + return Impl::atomic_oper_fetch(Impl::OrOper(),dest,val); +} + +template < typename T > +KOKKOS_INLINE_FUNCTION +T atomic_xor_fetch(volatile T * const dest, const T val) { + return Impl::atomic_oper_fetch(Impl::XorOper(),dest,val); +} + +template < typename T > +KOKKOS_INLINE_FUNCTION +T atomic_lshift_fetch(volatile T * const dest, const unsigned int val) { + return Impl::atomic_oper_fetch(Impl::LShiftOper(),dest,val); +} + +template < typename T > +KOKKOS_INLINE_FUNCTION +T atomic_rshift_fetch(volatile T * const dest, const unsigned int val) { + return Impl::atomic_oper_fetch(Impl::RShiftOper(),dest,val); +} + + +} +#endif diff --git a/lib/kokkos/core/src/impl/Kokkos_Atomic_View.hpp b/lib/kokkos/core/src/impl/Kokkos_Atomic_View.hpp new file mode 100755 index 0000000000..6bb33f6bfd --- /dev/null +++ b/lib/kokkos/core/src/impl/Kokkos_Atomic_View.hpp @@ -0,0 +1,448 @@ +/* +//@HEADER +// ************************************************************************ +// +// Kokkos: Manycore Performance-Portable Multidimensional Arrays +// Copyright (2012) Sandia Corporation +// +// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +// the U.S. Government retains certain rights in this software. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// 1. Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// 3. Neither the name of the Corporation nor the names of the +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Questions? Contact H. Carter Edwards (hcedwar@sandia.gov) +// +// ************************************************************************ +//@HEADER +*/ +#ifndef KOKKOS_ATOMIC_VIEW_HPP +#define KOKKOS_ATOMIC_VIEW_HPP + +#include +#include +namespace Kokkos { +namespace Impl { + +//The following tag is used to prevent an implicit call of the constructor when trying +//to assign a literal 0 int ( = 0 ); +struct AtomicViewConstTag {}; + +template +class AtomicDataElement { +public: + typedef typename ViewTraits::value_type value_type; + typedef typename ViewTraits::const_value_type const_value_type; + typedef typename ViewTraits::non_const_value_type non_const_value_type; + volatile value_type* const ptr; + + KOKKOS_INLINE_FUNCTION + AtomicDataElement(value_type* ptr_, AtomicViewConstTag ):ptr(ptr_){} + + KOKKOS_INLINE_FUNCTION + const_value_type operator = (const_value_type& val) const { + *ptr = val; + return val; + } + KOKKOS_INLINE_FUNCTION + const_value_type operator = (volatile const_value_type& val) const { + *ptr = val; + return val; + } + + KOKKOS_INLINE_FUNCTION + void inc() const { + Kokkos::atomic_increment(ptr); + } + + KOKKOS_INLINE_FUNCTION + void dec() const { + Kokkos::atomic_decrement(ptr); + } + + KOKKOS_INLINE_FUNCTION + const_value_type operator ++ () const { + const_value_type tmp = Kokkos::atomic_fetch_add(ptr,1); + return tmp+1; + } + + KOKKOS_INLINE_FUNCTION + const_value_type operator -- () const { + const_value_type tmp = Kokkos::atomic_fetch_add(ptr,-1); + return tmp-1; + } + + KOKKOS_INLINE_FUNCTION + const_value_type operator ++ (int) const { + return Kokkos::atomic_fetch_add(ptr,1); + } + + KOKKOS_INLINE_FUNCTION + const_value_type operator -- (int) const { + return Kokkos::atomic_fetch_add(ptr,-1); + } + + KOKKOS_INLINE_FUNCTION + const_value_type operator += (const_value_type& val) const { + const_value_type tmp = Kokkos::atomic_fetch_add(ptr,val); + return tmp+val; + } + KOKKOS_INLINE_FUNCTION + const_value_type operator += (volatile const_value_type& val) const { + const_value_type tmp = Kokkos::atomic_fetch_add(ptr,val); + return tmp+val; + } + + KOKKOS_INLINE_FUNCTION + const_value_type operator -= (const_value_type& val) const { + const_value_type tmp = Kokkos::atomic_fetch_add(ptr,-val); + return tmp-val; + } + KOKKOS_INLINE_FUNCTION + const_value_type operator -= (volatile const_value_type& val) const { + const_value_type tmp = Kokkos::atomic_fetch_add(ptr,-val); + return tmp-val; + } + + KOKKOS_INLINE_FUNCTION + const_value_type operator *= (const_value_type& val) const { + return Kokkos::atomic_mul_fetch(ptr,val); + } + KOKKOS_INLINE_FUNCTION + const_value_type operator *= (volatile const_value_type& val) const { + return Kokkos::atomic_mul_fetch(ptr,val); + } + + KOKKOS_INLINE_FUNCTION + const_value_type operator /= (const_value_type& val) const { + return Kokkos::atomic_div_fetch(ptr,val); + } + KOKKOS_INLINE_FUNCTION + const_value_type operator /= (volatile const_value_type& val) const { + return Kokkos::atomic_div_fetch(ptr,val); + } + + KOKKOS_INLINE_FUNCTION + const_value_type operator %= (const_value_type& val) const { + return Kokkos::atomic_mod_fetch(ptr,val); + } + KOKKOS_INLINE_FUNCTION + const_value_type operator %= (volatile const_value_type& val) const { + return Kokkos::atomic_mod_fetch(ptr,val); + } + + KOKKOS_INLINE_FUNCTION + const_value_type operator &= (const_value_type& val) const { + return Kokkos::atomic_and_fetch(ptr,val); + } + KOKKOS_INLINE_FUNCTION + const_value_type operator &= (volatile const_value_type& val) const { + return Kokkos::atomic_and_fetch(ptr,val); + } + + KOKKOS_INLINE_FUNCTION + const_value_type operator ^= (const_value_type& val) const { + return Kokkos::atomic_xor_fetch(ptr,val); + } + KOKKOS_INLINE_FUNCTION + const_value_type operator ^= (volatile const_value_type& val) const { + return Kokkos::atomic_xor_fetch(ptr,val); + } + + KOKKOS_INLINE_FUNCTION + const_value_type operator |= (const_value_type& val) const { + return Kokkos::atomic_or_fetch(ptr,val); + } + KOKKOS_INLINE_FUNCTION + const_value_type operator |= (volatile const_value_type& val) const { + return Kokkos::atomic_or_fetch(ptr,val); + } + + KOKKOS_INLINE_FUNCTION + const_value_type operator <<= (const_value_type& val) const { + return Kokkos::atomic_lshift_fetch(ptr,val); + } + KOKKOS_INLINE_FUNCTION + const_value_type operator <<= (volatile const_value_type& val) const { + return Kokkos::atomic_lshift_fetch(ptr,val); + } + + KOKKOS_INLINE_FUNCTION + const_value_type operator >>= (const_value_type& val) const { + return Kokkos::atomic_rshift_fetch(ptr,val); + } + KOKKOS_INLINE_FUNCTION + const_value_type operator >>= (volatile const_value_type& val) const { + return Kokkos::atomic_rshift_fetch(ptr,val); + } + + KOKKOS_INLINE_FUNCTION + const_value_type operator + (const_value_type& val) const { + return *ptr+val; + } + KOKKOS_INLINE_FUNCTION + const_value_type operator + (volatile const_value_type& val) const { + return *ptr+val; + } + + KOKKOS_INLINE_FUNCTION + const_value_type operator - (const_value_type& val) const { + return *ptr-val; + } + KOKKOS_INLINE_FUNCTION + const_value_type operator - (volatile const_value_type& val) const { + return *ptr-val; + } + + KOKKOS_INLINE_FUNCTION + const_value_type operator * (const_value_type& val) const { + return *ptr*val; + } + KOKKOS_INLINE_FUNCTION + const_value_type operator * (volatile const_value_type& val) const { + return *ptr*val; + } + + KOKKOS_INLINE_FUNCTION + const_value_type operator / (const_value_type& val) const { + return *ptr/val; + } + KOKKOS_INLINE_FUNCTION + const_value_type operator / (volatile const_value_type& val) const { + return *ptr/val; + } + + KOKKOS_INLINE_FUNCTION + const_value_type operator % (const_value_type& val) const { + return *ptr^val; + } + KOKKOS_INLINE_FUNCTION + const_value_type operator % (volatile const_value_type& val) const { + return *ptr^val; + } + + KOKKOS_INLINE_FUNCTION + const_value_type operator ! () const { + return !*ptr; + } + + KOKKOS_INLINE_FUNCTION + const_value_type operator && (const_value_type& val) const { + return *ptr&&val; + } + KOKKOS_INLINE_FUNCTION + const_value_type operator && (volatile const_value_type& val) const { + return *ptr&&val; + } + + KOKKOS_INLINE_FUNCTION + const_value_type operator || (const_value_type& val) const { + return *ptr|val; + } + KOKKOS_INLINE_FUNCTION + const_value_type operator || (volatile const_value_type& val) const { + return *ptr|val; + } + + KOKKOS_INLINE_FUNCTION + const_value_type operator & (const_value_type& val) const { + return *ptr&val; + } + KOKKOS_INLINE_FUNCTION + const_value_type operator & (volatile const_value_type& val) const { + return *ptr&val; + } + + KOKKOS_INLINE_FUNCTION + const_value_type operator | (const_value_type& val) const { + return *ptr|val; + } + KOKKOS_INLINE_FUNCTION + const_value_type operator | (volatile const_value_type& val) const { + return *ptr|val; + } + + KOKKOS_INLINE_FUNCTION + const_value_type operator ^ (const_value_type& val) const { + return *ptr^val; + } + KOKKOS_INLINE_FUNCTION + const_value_type operator ^ (volatile const_value_type& val) const { + return *ptr^val; + } + + KOKKOS_INLINE_FUNCTION + const_value_type operator ~ () const { + return ~*ptr; + } + + KOKKOS_INLINE_FUNCTION + const_value_type operator << (const unsigned int& val) const { + return *ptr<> (const unsigned int& val) const { + return *ptr>>val; + } + KOKKOS_INLINE_FUNCTION + const_value_type operator >> (volatile const unsigned int& val) const { + return *ptr>>val; + } + + KOKKOS_INLINE_FUNCTION + bool operator == (const_value_type& val) const { + return *ptr == val; + } + KOKKOS_INLINE_FUNCTION + bool operator == (volatile const_value_type& val) const { + return *ptr == val; + } + + KOKKOS_INLINE_FUNCTION + bool operator != (const_value_type& val) const { + return *ptr != val; + } + KOKKOS_INLINE_FUNCTION + bool operator != (volatile const_value_type& val) const { + return *ptr != val; + } + + KOKKOS_INLINE_FUNCTION + bool operator >= (const_value_type& val) const { + return *ptr >= val; + } + KOKKOS_INLINE_FUNCTION + bool operator >= (volatile const_value_type& val) const { + return *ptr >= val; + } + + KOKKOS_INLINE_FUNCTION + bool operator <= (const_value_type& val) const { + return *ptr <= val; + } + KOKKOS_INLINE_FUNCTION + bool operator <= (volatile const_value_type& val) const { + return *ptr <= val; + } + + KOKKOS_INLINE_FUNCTION + bool operator < (const_value_type& val) const { + return *ptr < val; + } + KOKKOS_INLINE_FUNCTION + bool operator < (volatile const_value_type& val) const { + return *ptr < val; + } + + KOKKOS_INLINE_FUNCTION + bool operator > (const_value_type& val) const { + return *ptr > val; + } + KOKKOS_INLINE_FUNCTION + bool operator > (volatile const_value_type& val) const { + return *ptr > val; + } + + KOKKOS_INLINE_FUNCTION + operator const_value_type () const { + //return Kokkos::atomic_load(ptr); + return *ptr; + } + + KOKKOS_INLINE_FUNCTION + operator volatile non_const_value_type () volatile const { + //return Kokkos::atomic_load(ptr); + return *ptr; + } +}; + +template +class AtomicViewDataHandle { +public: + typename ViewTraits::value_type* ptr; + + KOKKOS_INLINE_FUNCTION + AtomicViewDataHandle(typename ViewTraits::value_type* ptr_):ptr(ptr_){} + + template + KOKKOS_INLINE_FUNCTION + AtomicDataElement operator[] (const iType& i) const { + return AtomicDataElement(ptr+i,AtomicViewConstTag()); + } + + + KOKKOS_INLINE_FUNCTION + operator typename ViewTraits::value_type * () const { return ptr ; } + +}; + +template +struct Kokkos_Atomic_is_only_allowed_with_32bit_and_64bit_scalars; + +template<> +struct Kokkos_Atomic_is_only_allowed_with_32bit_and_64bit_scalars<4> { + typedef int type; +}; + +template<> +struct Kokkos_Atomic_is_only_allowed_with_32bit_and_64bit_scalars<8> { + typedef int64_t type; +}; + +// Must be non-const, atomic access trait, and 32 or 64 bit type for true atomics. +template +class ViewDataHandle< + ViewTraits , + typename enable_if< + ( ! is_same::value) && + ( ViewTraits::memory_traits::Atomic ) + >::type > +{ +private: +// typedef typename if_c<(sizeof(typename ViewTraits::const_value_type)==4) || +// (sizeof(typename ViewTraits::const_value_type)==8), +// int, Kokkos_Atomic_is_only_allowed_with_32bit_and_64bit_scalars >::type +// atomic_view_possible; + typedef typename Kokkos_Atomic_is_only_allowed_with_32bit_and_64bit_scalars::type enable_atomic_type; + typedef ViewDataHandle self_type; + +public: + enum { ReturnTypeIsReference = false }; + + typedef Impl::AtomicViewDataHandle handle_type; + typedef Impl::AtomicDataElement return_type; +}; + +} +} + +#endif diff --git a/lib/kokkos/core/src/impl/Kokkos_Core.cpp b/lib/kokkos/core/src/impl/Kokkos_Core.cpp new file mode 100755 index 0000000000..25542fa3d0 --- /dev/null +++ b/lib/kokkos/core/src/impl/Kokkos_Core.cpp @@ -0,0 +1,441 @@ +/* +//@HEADER +// ************************************************************************ +// +// Kokkos +// Manycore Performance-Portable Multidimensional Arrays +// +// Copyright (2012) Sandia Corporation +// +// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +// the U.S. Government retains certain rights in this software. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// 1. Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// 3. Neither the name of the Corporation nor the names of the +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Questions? Contact H. Carter Edwards (hcedwar@sandia.gov) +// +// ************************************************************************ +//@HEADER +*/ + +#include +#include +#include +#include +#include +#include + +//---------------------------------------------------------------------------- + +namespace Kokkos { +namespace Impl { +namespace { + +bool is_unsigned_int(const char* str) +{ + const size_t len = strlen (str); + for (size_t i = 0; i < len; ++i) { + if (! isdigit (str[i])) { + return false; + } + } + return true; +} + +void initialize_internal(const InitArguments& args) +{ + // Protect declarations, to prevent "unused variable" warnings. +#if defined( KOKKOS_HAVE_OPENMP ) || defined( KOKKOS_HAVE_PTHREAD ) + const int num_threads = args.num_threads; + const int use_numa = args.num_numa; +#endif // defined( KOKKOS_HAVE_OPENMP ) || defined( KOKKOS_HAVE_PTHREAD ) +#if defined( KOKKOS_HAVE_CUDA ) + const int use_gpu = args.device_id; +#endif // defined( KOKKOS_HAVE_CUDA ) + +#if defined( KOKKOS_HAVE_OPENMP ) + if( Impl::is_same< Kokkos::OpenMP , Kokkos::DefaultExecutionSpace >::value || + Impl::is_same< Kokkos::OpenMP , Kokkos::HostSpace::execution_space >::value ) { + if(num_threads>0) { + if(use_numa>0) { + Kokkos::OpenMP::initialize(num_threads,use_numa); + } + else { + Kokkos::OpenMP::initialize(num_threads); + } + } else { + Kokkos::OpenMP::initialize(); + } + //std::cout << "Kokkos::initialize() fyi: OpenMP enabled and initialized" << std::endl ; + } + else { + //std::cout << "Kokkos::initialize() fyi: OpenMP enabled but not initialized" << std::endl ; + } +#endif + +#if defined( KOKKOS_HAVE_PTHREAD ) + if( Impl::is_same< Kokkos::Threads , Kokkos::DefaultExecutionSpace >::value || + Impl::is_same< Kokkos::Threads , Kokkos::HostSpace::execution_space >::value ) { + if(num_threads>0) { + if(use_numa>0) { + Kokkos::Threads::initialize(num_threads,use_numa); + } + else { + Kokkos::Threads::initialize(num_threads); + } + } else { + Kokkos::Threads::initialize(); + } + //std::cout << "Kokkos::initialize() fyi: Pthread enabled and initialized" << std::endl ; + } + else { + //std::cout << "Kokkos::initialize() fyi: Pthread enabled but not initialized" << std::endl ; + } +#endif + +#if defined( KOKKOS_HAVE_SERIAL ) + // Prevent "unused variable" warning for 'args' input struct. If + // Serial::initialize() ever needs to take arguments from the input + // struct, you may remove this line of code. + (void) args; + + if( Impl::is_same< Kokkos::Serial , Kokkos::DefaultExecutionSpace >::value || + Impl::is_same< Kokkos::Serial , Kokkos::HostSpace::execution_space >::value ) { + Kokkos::Serial::initialize(); + } +#endif + +#if defined( KOKKOS_HAVE_CUDA ) + if( Impl::is_same< Kokkos::Cuda , Kokkos::DefaultExecutionSpace >::value || 0 < use_gpu ) { + if (use_gpu > -1) { + Kokkos::Cuda::initialize( Kokkos::Cuda::SelectDevice( use_gpu ) ); + } + else { + Kokkos::Cuda::initialize(); + } + //std::cout << "Kokkos::initialize() fyi: Cuda enabled and initialized" << std::endl ; + } +#endif +} + +void finalize_internal( const bool all_spaces = false ) +{ + +#if defined( KOKKOS_HAVE_CUDA ) + if( Impl::is_same< Kokkos::Cuda , Kokkos::DefaultExecutionSpace >::value || all_spaces ) { + if(Kokkos::Cuda::is_initialized()) + Kokkos::Cuda::finalize(); + } +#endif + +#if defined( KOKKOS_HAVE_OPENMP ) + if( Impl::is_same< Kokkos::OpenMP , Kokkos::DefaultExecutionSpace >::value || + Impl::is_same< Kokkos::OpenMP , Kokkos::HostSpace::execution_space >::value || + all_spaces ) { + if(Kokkos::OpenMP::is_initialized()) + Kokkos::OpenMP::finalize(); + } +#endif + +#if defined( KOKKOS_HAVE_PTHREAD ) + if( Impl::is_same< Kokkos::Threads , Kokkos::DefaultExecutionSpace >::value || + Impl::is_same< Kokkos::Threads , Kokkos::HostSpace::execution_space >::value || + all_spaces ) { + if(Kokkos::Threads::is_initialized()) + Kokkos::Threads::finalize(); + } +#endif + +#if defined( KOKKOS_HAVE_SERIAL ) + if( Impl::is_same< Kokkos::Serial , Kokkos::DefaultExecutionSpace >::value || + Impl::is_same< Kokkos::Serial , Kokkos::HostSpace::execution_space >::value || + all_spaces ) { + if(Kokkos::Serial::is_initialized()) + Kokkos::Serial::finalize(); + } +#endif + +} + +void fence_internal() +{ + +#if defined( KOKKOS_HAVE_CUDA ) + if( Impl::is_same< Kokkos::Cuda , Kokkos::DefaultExecutionSpace >::value ) { + Kokkos::Cuda::fence(); + } +#endif + +#if defined( KOKKOS_HAVE_OPENMP ) + if( Impl::is_same< Kokkos::OpenMP , Kokkos::DefaultExecutionSpace >::value || + Impl::is_same< Kokkos::OpenMP , Kokkos::HostSpace::execution_space >::value ) { + Kokkos::OpenMP::fence(); + } +#endif + +#if defined( KOKKOS_HAVE_PTHREAD ) + if( Impl::is_same< Kokkos::Threads , Kokkos::DefaultExecutionSpace >::value || + Impl::is_same< Kokkos::Threads , Kokkos::HostSpace::execution_space >::value ) { + Kokkos::Threads::fence(); + } +#endif + +#if defined( KOKKOS_HAVE_SERIAL ) + if( Impl::is_same< Kokkos::Serial , Kokkos::DefaultExecutionSpace >::value || + Impl::is_same< Kokkos::Serial , Kokkos::HostSpace::execution_space >::value ) { + Kokkos::Serial::fence(); + } +#endif + +} + +} // namespace +} // namespace Impl +} // namespace Kokkos + +//---------------------------------------------------------------------------- + +namespace Kokkos { + +void initialize(int& narg, char* arg[]) +{ + int num_threads = -1; + int numa = -1; + int device = -1; + + int kokkos_threads_found = 0; + int kokkos_numa_found = 0; + int kokkos_device_found = 0; + int kokkos_ndevices_found = 0; + + int iarg = 0; + + while (iarg < narg) { + if ((strncmp(arg[iarg],"--kokkos-threads",16) == 0) || (strncmp(arg[iarg],"--threads",9) == 0)) { + //Find the number of threads (expecting --threads=XX) + if (!((strncmp(arg[iarg],"--kokkos-threads=",17) == 0) || (strncmp(arg[iarg],"--threads=",10) == 0))) + Impl::throw_runtime_exception("Error: expecting an '=INT' after command line argument '--threads/--kokkos-threads'. Raised by Kokkos::initialize(int narg, char* argc[])."); + + char* number = strchr(arg[iarg],'=')+1; + + if(!Impl::is_unsigned_int(number) || (strlen(number)==0)) + Impl::throw_runtime_exception("Error: expecting an '=INT' after command line argument '--threads/--kokkos-threads'. Raised by Kokkos::initialize(int narg, char* argc[])."); + + if((strncmp(arg[iarg],"--kokkos-threads",16) == 0) || !kokkos_threads_found) + num_threads = atoi(number); + + //Remove the --kokkos-threads argument from the list but leave --threads + if(strncmp(arg[iarg],"--kokkos-threads",16) == 0) { + for(int k=iarg;k= skip_device) device++; + } + if ((str = getenv("MV2_COMM_WORLD_LOCAL_RANK"))) { + int local_rank = atoi(str); + device = local_rank % ndevices; + if (device >= skip_device) device++; + } + if ((str = getenv("OMPI_COMM_WORLD_LOCAL_RANK"))) { + int local_rank = atoi(str); + device = local_rank % ndevices; + if (device >= skip_device) device++; + } + if(device==-1) { + device = 0; + if (device >= skip_device) device++; + } + } + + //Remove the --kokkos-ndevices argument from the list but leave --ndevices + if(strncmp(arg[iarg],"--kokkos-ndevices",17) == 0) { + for(int k=iarg;k +inline +typename CrsArray< DataType , Arg1Type , Arg2Type , SizeType >::HostMirror +create_mirror( const CrsArray & view ) +{ + // Force copy: + //typedef Impl::ViewAssignment< Impl::ViewDefault > alloc ; // unused + typedef CrsArray< DataType , Arg1Type , Arg2Type , SizeType > crsarray_type ; + + typename crsarray_type::HostMirror tmp ; + typename crsarray_type::row_map_type::HostMirror tmp_row_map = create_mirror( view.row_map ); + + tmp.row_map = tmp_row_map ; // Assignment of 'const' from 'non-const' + tmp.entries = create_mirror( view.entries ); + + // Deep copy: + deep_copy( tmp_row_map , view.row_map ); + deep_copy( tmp.entries , view.entries ); + + return tmp ; +} + +template< class DataType , class Arg1Type , class Arg2Type , typename SizeType > +inline +typename CrsArray< DataType , Arg1Type , Arg2Type , SizeType >::HostMirror +create_mirror_view( const CrsArray & view , + typename Impl::enable_if< ViewTraits::is_hostspace >::type * = 0 ) +{ + return view ; +} + +template< class DataType , class Arg1Type , class Arg2Type , typename SizeType > +inline +typename CrsArray< DataType , Arg1Type , Arg2Type , SizeType >::HostMirror +create_mirror_view( const CrsArray & view , + typename Impl::enable_if< ! ViewTraits::is_hostspace >::type * = 0 ) +{ + return create_mirror( view ); +} + + +} // namespace Kokkos + +//---------------------------------------------------------------------------- +//---------------------------------------------------------------------------- + +namespace Kokkos { + +template< class CrsArrayType , class InputSizeType > +inline +typename CrsArrayType::crsarray_type +create_crsarray( const std::string & label , + const std::vector< InputSizeType > & input ) +{ + typedef CrsArrayType output_type ; + //typedef std::vector< InputSizeType > input_type ; // unused + + typedef typename output_type::entries_type entries_type ; + + typedef View< typename output_type::size_type [] , + typename output_type::array_layout , + typename output_type::execution_space > work_type ; + + output_type output ; + + // Create the row map: + + const size_t length = input.size(); + + { + work_type row_work( "tmp" , length + 1 ); + + typename work_type::HostMirror row_work_host = + create_mirror_view( row_work ); + + size_t sum = 0 ; + row_work_host[0] = 0 ; + for ( size_t i = 0 ; i < length ; ++i ) { + row_work_host[i+1] = sum += input[i]; + } + + deep_copy( row_work , row_work_host ); + + output.entries = entries_type( label , sum ); + output.row_map = row_work ; + } + + return output ; +} + +//---------------------------------------------------------------------------- + +template< class CrsArrayType , class InputSizeType > +inline +typename CrsArrayType::crsarray_type +create_crsarray( const std::string & label , + const std::vector< std::vector< InputSizeType > > & input ) +{ + typedef CrsArrayType output_type ; + //typedef std::vector< std::vector< InputSizeType > > input_type ; // unused + typedef typename output_type::entries_type entries_type ; + //typedef typename output_type::size_type size_type ; // unused + + // mfh 14 Feb 2014: This function doesn't actually create instances + // of ok_rank, but it needs to declare the typedef in order to do + // the static "assert" (a compile-time check that the given shape + // has rank 1). In order to avoid a "declared but unused typedef" + // warning, we declare an empty instance of this type, with the + // usual "(void)" marker to avoid a compiler warning for the unused + // variable. + + typedef typename + Impl::assert_shape_is_rank_one< typename entries_type::shape_type >::type + ok_rank ; + { + ok_rank thing; + (void) thing; + } + + typedef View< typename output_type::size_type [] , + typename output_type::array_layout , + typename output_type::execution_space > work_type ; + + output_type output ; + + // Create the row map: + + const size_t length = input.size(); + + { + work_type row_work( "tmp" , length + 1 ); + + typename work_type::HostMirror row_work_host = + create_mirror_view( row_work ); + + size_t sum = 0 ; + row_work_host[0] = 0 ; + for ( size_t i = 0 ; i < length ; ++i ) { + row_work_host[i+1] = sum += input[i].size(); + } + + deep_copy( row_work , row_work_host ); + + output.entries = entries_type( label , sum ); + output.row_map = row_work ; + } + + // Fill in the entries: + { + typename entries_type::HostMirror host_entries = + create_mirror_view( output.entries ); + + size_t sum = 0 ; + for ( size_t i = 0 ; i < length ; ++i ) { + for ( size_t j = 0 ; j < input[i].size() ; ++j , ++sum ) { + host_entries( sum ) = input[i][j] ; + } + } + + deep_copy( output.entries , host_entries ); + } + + return output ; +} + +} // namespace Kokkos + +//---------------------------------------------------------------------------- +//---------------------------------------------------------------------------- + +#endif /* #ifndef KOKKOS_IMPL_CRSARRAY_FACTORY_HPP */ + diff --git a/lib/kokkos/core/src/impl/Kokkos_Error.cpp b/lib/kokkos/core/src/impl/Kokkos_Error.cpp new file mode 100755 index 0000000000..00fe438845 --- /dev/null +++ b/lib/kokkos/core/src/impl/Kokkos_Error.cpp @@ -0,0 +1,195 @@ +/* +//@HEADER +// ************************************************************************ +// +// Kokkos +// Manycore Performance-Portable Multidimensional Arrays +// +// Copyright (2012) Sandia Corporation +// +// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +// the U.S. Government retains certain rights in this software. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// 1. Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// 3. Neither the name of the Corporation nor the names of the +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Questions? Contact H. Carter Edwards (hcedwar@sandia.gov) +// +// ************************************************************************ +//@HEADER +*/ + +#include +#include +#include + +#include +#include +#include +#include +#include + +//---------------------------------------------------------------------------- +//---------------------------------------------------------------------------- + +namespace Kokkos { +namespace Impl { + +void host_abort( const char * const message ) +{ + fwrite(message,1,strlen(message),stderr); + fflush(stderr); + abort(); +} + +void throw_runtime_exception( const std::string & msg ) +{ + std::ostringstream o ; + o << msg ; + traceback_callstack( o ); + throw std::runtime_error( o.str() ); +} + + +std::string human_memory_size(size_t arg_bytes) +{ + double bytes = arg_bytes; + const double K = 1024; + const double M = K*1024; + const double G = M*1024; + + std::ostringstream out; + if (bytes < K) { + out << std::setprecision(4) << bytes << " B"; + } else if (bytes < M) { + bytes /= K; + out << std::setprecision(4) << bytes << " K"; + } else if (bytes < G) { + bytes /= M; + out << std::setprecision(4) << bytes << " M"; + } else { + bytes /= G; + out << std::setprecision(4) << bytes << " G"; + } + return out.str(); +} + +} +} + +//---------------------------------------------------------------------------- +//---------------------------------------------------------------------------- + +#if defined( __GNUC__ ) && defined( ENABLE_TRACEBACK ) + +/* This is only known to work with GNU C++ + * Must be compiled with '-rdynamic' + * Must be linked with '-ldl' + */ + +/* Print call stack into an error stream, + * so one knows in which function the error occured. + * + * Code copied from: + * http://stupefydeveloper.blogspot.com/2008/10/cc-call-stack.html + * + * License on this site: + * This blog is licensed under a + * Creative Commons Attribution-Share Alike 3.0 Unported License. + * + * http://creativecommons.org/licenses/by-sa/3.0/ + * + * Modified to output to std::ostream. + */ +#include +#include +#include +#include +#include + +namespace Kokkos { +namespace Impl { + +void traceback_callstack( std::ostream & msg ) +{ + using namespace abi; + + enum { MAX_DEPTH = 32 }; + + void *trace[MAX_DEPTH]; + Dl_info dlinfo; + + int status; + + int trace_size = backtrace(trace, MAX_DEPTH); + + msg << std::endl << "Call stack {" << std::endl ; + + for (int i=1; i +#include + +namespace Kokkos { +namespace Impl { + +void host_abort( const char * const ); + +void throw_runtime_exception( const std::string & ); + +void traceback_callstack( std::ostream & ); + +std::string human_memory_size(size_t arg_bytes); + +} +} + +//---------------------------------------------------------------------------- +//---------------------------------------------------------------------------- + +#if defined( KOKKOS_ACTIVE_EXECUTION_MEMORY_SPACE_HOST ) +namespace Kokkos { +inline +void abort( const char * const message ) { Kokkos::Impl::host_abort(message); } +} +#endif /* defined( KOKKOS_ACTIVE_EXECUTION_MEMORY_SPACE_CUDA ) */ + +//---------------------------------------------------------------------------- +//---------------------------------------------------------------------------- + +#endif /* #ifndef KOKKOS_IMPL_ERROR_HPP */ + diff --git a/lib/kokkos/core/src/impl/Kokkos_FunctorAdapter.hpp b/lib/kokkos/core/src/impl/Kokkos_FunctorAdapter.hpp new file mode 100755 index 0000000000..fb5add6a72 --- /dev/null +++ b/lib/kokkos/core/src/impl/Kokkos_FunctorAdapter.hpp @@ -0,0 +1,960 @@ +/* +//@HEADER +// ************************************************************************ +// +// Kokkos: Manycore Performance-Portable Multidimensional Arrays +// Copyright (2012) Sandia Corporation +// +// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +// the U.S. Government retains certain rights in this software. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// 1. Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// 3. Neither the name of the Corporation nor the names of the +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Questions? Contact H. Carter Edwards (hcedwar@sandia.gov) +// +// ************************************************************************ +//@HEADER +*/ + +#ifndef KOKKOS_FUNCTORADAPTER_HPP +#define KOKKOS_FUNCTORADAPTER_HPP + +#include +#include +#include +#include + +//---------------------------------------------------------------------------- +//---------------------------------------------------------------------------- + +namespace Kokkos { +namespace Impl { + +template< class FunctorType , class ArgTag , class Enable = void > +struct FunctorDeclaresValueType : public Impl::false_type {}; + +template< class FunctorType , class ArgTag > +struct FunctorDeclaresValueType< FunctorType , ArgTag + , typename Impl::enable_if_type< typename FunctorType::value_type >::type > + : public Impl::true_type {}; + + +/** \brief Query Functor and execution policy argument tag for value type. + * + * If C++11 enabled and 'value_type' is not explicitly declared then attempt + * to deduce the type from FunctorType::operator(). + */ +template< class FunctorType , class ArgTag , bool Dec = FunctorDeclaresValueType::value > +struct FunctorValueTraits +{ + typedef void value_type ; + typedef void pointer_type ; + typedef void reference_type ; + + enum { StaticValueSize = 0 }; + + KOKKOS_FORCEINLINE_FUNCTION static + unsigned value_count( const FunctorType & ) { return 0 ; } + + KOKKOS_FORCEINLINE_FUNCTION static + unsigned value_size( const FunctorType & ) { return 0 ; } +}; + +/** \brief FunctorType::value_type is explicitly declared so use it. + * + * Two options for declaration + * + * 1) A plain-old-data (POD) type + * typedef {pod_type} value_type ; + * + * 2) An array of POD of a runtime specified count. + * typedef {pod_type} value_type[] ; + * const unsigned value_count ; + */ +template< class FunctorType , class ArgTag > +struct FunctorValueTraits< FunctorType , ArgTag , true /* exists FunctorType::value_type */ > +{ + typedef typename Impl::remove_extent< typename FunctorType::value_type >::type value_type ; + + // If not an array then what is the sizeof(value_type) + enum { StaticValueSize = Impl::is_array< typename FunctorType::value_type >::value ? 0 : sizeof(value_type) }; + + typedef value_type * pointer_type ; + + // The reference_type for an array is 'value_type *' + // The reference_type for a single value is 'value_type &' + + typedef typename Impl::if_c< ! StaticValueSize , value_type * + , value_type & >::type reference_type ; + + // Number of values if single value + template< class F > + KOKKOS_FORCEINLINE_FUNCTION static + typename Impl::enable_if< Impl::is_same::value && StaticValueSize , unsigned >::type + value_count( const F & ) { return 1 ; } + + // Number of values if an array, protect via templating because 'f.value_count' + // will only exist when the functor declares the value_type to be an array. + template< class F > + KOKKOS_FORCEINLINE_FUNCTION static + typename Impl::enable_if< Impl::is_same::value && ! StaticValueSize , unsigned >::type + value_count( const F & f ) { return f.value_count ; } + + // Total size of the value + KOKKOS_INLINE_FUNCTION static + unsigned value_size( const FunctorType & f ) { return value_count( f ) * sizeof(value_type) ; } +}; + + +#if defined( KOKKOS_HAVE_CXX11 ) + +// If have C++11 and functor does not explicitly specify a value type +// then try to deduce the value type from FunctorType::operator(). +// Can only deduce single value type since array length cannot be deduced. +template< class FunctorType > +struct FunctorValueTraits< FunctorType + , void /* == ArgTag */ + , false /* == exists FunctorType::value_type */ + > +{ +private: + + struct VOID {}; + + // parallel_for operator without a tag: + template< class ArgMember > + KOKKOS_INLINE_FUNCTION + static VOID deduce( void (FunctorType::*)( ArgMember ) const ) {} + + // parallel_reduce operator without a tag: + template< class ArgMember , class T > + KOKKOS_INLINE_FUNCTION + static T deduce( void (FunctorType::*)( ArgMember , T & ) const ) {} + + // parallel_scan operator without a tag: + template< class ArgMember , class T > + KOKKOS_INLINE_FUNCTION + static T deduce( void (FunctorType::*)( ArgMember , T & , bool ) const ) {} + + typedef decltype( deduce( & FunctorType::operator() ) ) ValueType ; + + enum { IS_VOID = Impl::is_same::value }; + +public: + + typedef typename Impl::if_c< IS_VOID , void , ValueType >::type value_type ; + typedef typename Impl::if_c< IS_VOID , void , ValueType * >::type pointer_type ; + typedef typename Impl::if_c< IS_VOID , void , ValueType & >::type reference_type ; + + enum { StaticValueSize = IS_VOID ? 0 : sizeof(ValueType) }; + + KOKKOS_FORCEINLINE_FUNCTION static + unsigned value_size( const FunctorType & ) { return StaticValueSize ; } + + KOKKOS_FORCEINLINE_FUNCTION static + unsigned value_count( const FunctorType & ) { return IS_VOID ? 0 : 1 ; } +}; + + +template< class FunctorType , class ArgTag > +struct FunctorValueTraits< FunctorType + , ArgTag /* != void */ + , false /* == exists FunctorType::value_type */ + > +{ +private: + + //---------------------------------------- + // parallel_for operator with a tag: + + struct VOID {}; // to allow valid sizeof(ValueType) + + template< class ArgMember > + KOKKOS_INLINE_FUNCTION + static VOID deduce( void (FunctorType::*)( ArgTag , ArgMember ) const ) {} + + template< class ArgMember > + KOKKOS_INLINE_FUNCTION + static VOID deduce( void (FunctorType::*)( const ArgTag & , ArgMember ) const ) {} + + //---------------------------------------- + // parallel_reduce operator with a tag: + + template< class ArgMember , class T > + KOKKOS_INLINE_FUNCTION + static T deduce( void (FunctorType::*)( ArgTag , ArgMember , T & ) const ) {} + + template< class ArgMember , class T > + KOKKOS_INLINE_FUNCTION + static T deduce( void (FunctorType::*)( const ArgTag & , ArgMember , T & ) const ) {} + + //---------------------------------------- + // parallel_scan operator with a tag: + + template< class ArgMember , class T > + KOKKOS_INLINE_FUNCTION + static T deduce( void (FunctorType::*)( ArgTag , ArgMember , T & , bool ) const ) {} + + template< class ArgMember , class T > + KOKKOS_INLINE_FUNCTION + static T deduce( void (FunctorType::*)( const ArgTag & , ArgMember , T & , bool ) const ) {} + + //---------------------------------------- + + typedef decltype( deduce( & FunctorType::operator() ) ) ValueType ; + + enum { IS_VOID = Impl::is_same::value }; + +public: + + typedef typename Impl::if_c< IS_VOID , void , ValueType >::type value_type ; + typedef typename Impl::if_c< IS_VOID , void , ValueType * >::type pointer_type ; + typedef typename Impl::if_c< IS_VOID , void , ValueType & >::type reference_type ; + + enum { StaticValueSize = IS_VOID ? 0 : sizeof(ValueType) }; + + KOKKOS_FORCEINLINE_FUNCTION static + unsigned value_size( const FunctorType & ) { return StaticValueSize ; } + + KOKKOS_FORCEINLINE_FUNCTION static + unsigned value_count( const FunctorType & ) { return IS_VOID ? 0 : 1 ; } +}; + +#endif /* #if defined( KOKKOS_HAVE_CXX11 ) */ + +} // namespace Impl +} // namespace Kokkos + +//---------------------------------------------------------------------------- +//---------------------------------------------------------------------------- + +namespace Kokkos { +namespace Impl { + +// Function signatures for FunctorType::init function with a tag and not an array +template< class FunctorType , class ArgTag , bool IsArray = 0 == FunctorValueTraits::StaticValueSize > +struct FunctorValueInitFunction { + + typedef typename FunctorValueTraits::value_type value_type ; + + KOKKOS_INLINE_FUNCTION static void enable_if( void (FunctorType::*)( ArgTag , value_type & ) const ); + KOKKOS_INLINE_FUNCTION static void enable_if( void (FunctorType::*)( ArgTag const & , value_type & ) const ); + KOKKOS_INLINE_FUNCTION static void enable_if( void ( *)( ArgTag , value_type & ) ); + KOKKOS_INLINE_FUNCTION static void enable_if( void ( *)( ArgTag const & , value_type & ) ); + + // KOKKOS_INLINE_FUNCTION static void enable_if( void (FunctorType::*)( ArgTag , value_type volatile & ) const ); + // KOKKOS_INLINE_FUNCTION static void enable_if( void (FunctorType::*)( ArgTag const & , value_type volatile & ) const ); + // KOKKOS_INLINE_FUNCTION static void enable_if( void ( *)( ArgTag , value_type volatile & ) ); + // KOKKOS_INLINE_FUNCTION static void enable_if( void ( *)( ArgTag const & , value_type volatile & ) ); +}; + +// Function signatures for FunctorType::init function with a tag and is an array +template< class FunctorType , class ArgTag > +struct FunctorValueInitFunction< FunctorType , ArgTag , true > { + + typedef typename FunctorValueTraits::value_type value_type ; + + KOKKOS_INLINE_FUNCTION static void enable_if( void (FunctorType::*)( ArgTag , value_type * ) const ); + KOKKOS_INLINE_FUNCTION static void enable_if( void (FunctorType::*)( ArgTag const & , value_type * ) const ); + KOKKOS_INLINE_FUNCTION static void enable_if( void ( *)( ArgTag , value_type * ) ); + KOKKOS_INLINE_FUNCTION static void enable_if( void ( *)( ArgTag const & , value_type * ) ); + + // KOKKOS_INLINE_FUNCTION static void enable_if( void (FunctorType::*)( ArgTag , value_type volatile * ) const ); + // KOKKOS_INLINE_FUNCTION static void enable_if( void (FunctorType::*)( ArgTag const & , value_type volatile * ) const ); + // KOKKOS_INLINE_FUNCTION static void enable_if( void ( *)( ArgTag , value_type volatile * ) ); + // KOKKOS_INLINE_FUNCTION static void enable_if( void ( *)( ArgTag const & , value_type volatile * ) ); +}; + +// Function signatures for FunctorType::init function without a tag and not an array +template< class FunctorType > +struct FunctorValueInitFunction< FunctorType , void , false > { + + typedef typename FunctorValueTraits::reference_type value_type ; + + KOKKOS_INLINE_FUNCTION static void enable_if( void (FunctorType::*)( value_type & ) const ); + KOKKOS_INLINE_FUNCTION static void enable_if( void ( *)( value_type & ) ); + + // KOKKOS_INLINE_FUNCTION static void enable_if( void (FunctorType::*)( value_type volatile & ) const ); + // KOKKOS_INLINE_FUNCTION static void enable_if( void ( *)( value_type volatile & ) ); +}; + +// Function signatures for FunctorType::init function without a tag and is an array +template< class FunctorType > +struct FunctorValueInitFunction< FunctorType , void , true > { + + typedef typename FunctorValueTraits::reference_type value_type ; + + KOKKOS_INLINE_FUNCTION static void enable_if( void (FunctorType::*)( value_type * ) const ); + KOKKOS_INLINE_FUNCTION static void enable_if( void ( *)( value_type * ) ); + + // KOKKOS_INLINE_FUNCTION static void enable_if( void (FunctorType::*)( value_type volatile * ) const ); + // KOKKOS_INLINE_FUNCTION static void enable_if( void ( *)( value_type volatile * ) ); +}; + +// Adapter for value initialization function. +// If a proper FunctorType::init is declared then use it, +// otherwise use default constructor. +template< class FunctorType , class ArgTag + , class T = typename FunctorValueTraits::reference_type + , class Enable = void > +struct FunctorValueInit ; + +/* No 'init' function provided for single value */ +template< class FunctorType , class ArgTag , class T , class Enable > +struct FunctorValueInit< FunctorType , ArgTag , T & , Enable > +{ + KOKKOS_FORCEINLINE_FUNCTION static + T & init( const FunctorType & f , void * p ) + { return *( new(p) T() ); }; +}; + +/* No 'init' function provided for array value */ +template< class FunctorType , class ArgTag , class T , class Enable > +struct FunctorValueInit< FunctorType , ArgTag , T * , Enable > +{ + KOKKOS_FORCEINLINE_FUNCTION static + T * init( const FunctorType & f , void * p ) + { + const int n = FunctorValueTraits< FunctorType , ArgTag >::value_count(f); + for ( int i = 0 ; i < n ; ++i ) { new( ((T*)p) + i ) T(); } + return (T*)p ; + } +}; + +/* 'init' function provided for single value */ +template< class FunctorType , class ArgTag , class T > +struct FunctorValueInit + < FunctorType + , ArgTag + , T & + // First substitution failure when FunctorType::init does not exist. +#if defined( KOKKOS_HAVE_CXX11 ) + // Second substitution failure when FunctorType::init is not compatible. + , decltype( FunctorValueInitFunction< FunctorType , ArgTag >::enable_if( & FunctorType::init ) ) +#else + , typename Impl::enable_if< 0 < sizeof( & FunctorType::init ) >::type +#endif + > +{ + KOKKOS_FORCEINLINE_FUNCTION static + T & init( const FunctorType & f , void * p ) + { f.init( *((T*)p) ); return *((T*)p) ; } +}; + +/* 'init' function provided for array value */ +template< class FunctorType , class ArgTag , class T > +struct FunctorValueInit + < FunctorType + , ArgTag + , T * + // First substitution failure when FunctorType::init does not exist. +#if defined( KOKKOS_HAVE_CXX11 ) + // Second substitution failure when FunctorType::init is not compatible + , decltype( FunctorValueInitFunction< FunctorType , ArgTag >::enable_if( & FunctorType::init ) ) +#else + , typename Impl::enable_if< 0 < sizeof( & FunctorType::init ) >::type +#endif + > +{ + KOKKOS_FORCEINLINE_FUNCTION static + T * init( const FunctorType & f , void * p ) + { f.init( (T*)p ); return (T*)p ; } +}; + +} // namespace Impl +} // namespace Kokkos + +//---------------------------------------------------------------------------- +//---------------------------------------------------------------------------- + +namespace Kokkos { +namespace Impl { + +// Signatures for compatible FunctorType::join with tag and not an array +template< class FunctorType , class ArgTag , bool IsArray = 0 == FunctorValueTraits::StaticValueSize > +struct FunctorValueJoinFunction { + + typedef typename FunctorValueTraits::value_type value_type ; + + typedef volatile value_type & vref_type ; + typedef const volatile value_type & cvref_type ; + + KOKKOS_INLINE_FUNCTION static void enable_if( void (FunctorType::*)( ArgTag , vref_type , cvref_type ) const ); + KOKKOS_INLINE_FUNCTION static void enable_if( void (FunctorType::*)( ArgTag const & , vref_type , cvref_type ) const ); + KOKKOS_INLINE_FUNCTION static void enable_if( void ( *)( ArgTag , vref_type , cvref_type ) ); + KOKKOS_INLINE_FUNCTION static void enable_if( void ( *)( ArgTag const & , vref_type , cvref_type ) ); +}; + +// Signatures for compatible FunctorType::join with tag and is an array +template< class FunctorType , class ArgTag > +struct FunctorValueJoinFunction< FunctorType , ArgTag , true > { + + typedef typename FunctorValueTraits::value_type value_type ; + + typedef volatile value_type * vptr_type ; + typedef const volatile value_type * cvptr_type ; + + KOKKOS_INLINE_FUNCTION static void enable_if( void (FunctorType::*)( ArgTag , vptr_type , cvptr_type ) const ); + KOKKOS_INLINE_FUNCTION static void enable_if( void (FunctorType::*)( ArgTag const & , vptr_type , cvptr_type ) const ); + KOKKOS_INLINE_FUNCTION static void enable_if( void ( *)( ArgTag , vptr_type , cvptr_type ) ); + KOKKOS_INLINE_FUNCTION static void enable_if( void ( *)( ArgTag const & , vptr_type , cvptr_type ) ); +}; + +// Signatures for compatible FunctorType::join without tag and not an array +template< class FunctorType > +struct FunctorValueJoinFunction< FunctorType , void , false > { + + typedef typename FunctorValueTraits::value_type value_type ; + + typedef volatile value_type & vref_type ; + typedef const volatile value_type & cvref_type ; + + KOKKOS_INLINE_FUNCTION static void enable_if( void (FunctorType::*)( vref_type , cvref_type ) const ); + KOKKOS_INLINE_FUNCTION static void enable_if( void ( *)( vref_type , cvref_type ) ); +}; + +// Signatures for compatible FunctorType::join without tag and is an array +template< class FunctorType > +struct FunctorValueJoinFunction< FunctorType , void , true > { + + typedef typename FunctorValueTraits::value_type value_type ; + + typedef volatile value_type * vptr_type ; + typedef const volatile value_type * cvptr_type ; + + KOKKOS_INLINE_FUNCTION static void enable_if( void (FunctorType::*)( vptr_type , cvptr_type ) const ); + KOKKOS_INLINE_FUNCTION static void enable_if( void ( *)( vptr_type , cvptr_type ) ); +}; + + +template< class FunctorType , class ArgTag + , class T = typename FunctorValueTraits::reference_type + , class Enable = void > +struct FunctorValueJoin ; + +/* No 'join' function provided, single value */ +template< class FunctorType , class ArgTag , class T , class Enable > +struct FunctorValueJoin< FunctorType , ArgTag , T & , Enable > +{ + KOKKOS_FORCEINLINE_FUNCTION static + void join( const FunctorType & f , volatile void * const lhs , const volatile void * const rhs ) + { + *((volatile T*)lhs) += *((const volatile T*)rhs); + } +}; + +/* No 'join' function provided, array of values */ +template< class FunctorType , class ArgTag , class T , class Enable > +struct FunctorValueJoin< FunctorType , ArgTag , T * , Enable > +{ + KOKKOS_FORCEINLINE_FUNCTION static + void join( const FunctorType & f , volatile void * const lhs , const volatile void * const rhs ) + { + const int n = FunctorValueTraits::value_count(f); + + for ( int i = 0 ; i < n ; ++i ) { ((volatile T*)lhs)[i] += ((const volatile T*)rhs)[i]; } + } +}; + +/* 'join' function provided, single value */ +template< class FunctorType , class ArgTag , class T > +struct FunctorValueJoin + < FunctorType + , ArgTag + , T & + // First substitution failure when FunctorType::join does not exist. +#if defined( KOKKOS_HAVE_CXX11 ) + // Second substitution failure when enable_if( & Functor::join ) does not exist + , decltype( FunctorValueJoinFunction< FunctorType , ArgTag >::enable_if( & FunctorType::join ) ) +#else + , typename Impl::enable_if< 0 < sizeof( & FunctorType::join ) >::type +#endif + > +{ + KOKKOS_FORCEINLINE_FUNCTION static + void join( const FunctorType & f , volatile void * const lhs , const volatile void * const rhs ) + { + f.join( ArgTag() , *((volatile T *)lhs) , *((const volatile T *)rhs) ); + } +}; + +/* 'join' function provided, no tag, single value */ +template< class FunctorType , class T > +struct FunctorValueJoin + < FunctorType + , void + , T & + // First substitution failure when FunctorType::join does not exist. +#if defined( KOKKOS_HAVE_CXX11 ) + // Second substitution failure when enable_if( & Functor::join ) does not exist + , decltype( FunctorValueJoinFunction< FunctorType , void >::enable_if( & FunctorType::join ) ) +#else + , typename Impl::enable_if< 0 < sizeof( & FunctorType::join ) >::type +#endif + > +{ + KOKKOS_FORCEINLINE_FUNCTION static + void join( const FunctorType & f , volatile void * const lhs , const volatile void * const rhs ) + { + f.join( *((volatile T *)lhs) , *((const volatile T *)rhs) ); + } +}; + +/* 'join' function provided for array value */ +template< class FunctorType , class ArgTag , class T > +struct FunctorValueJoin + < FunctorType + , ArgTag + , T * + // First substitution failure when FunctorType::join does not exist. +#if defined( KOKKOS_HAVE_CXX11 ) + // Second substitution failure when enable_if( & Functor::join ) does not exist + , decltype( FunctorValueJoinFunction< FunctorType , ArgTag >::enable_if( & FunctorType::join ) ) +#else + , typename Impl::enable_if< 0 < sizeof( & FunctorType::join ) >::type +#endif + > +{ + KOKKOS_FORCEINLINE_FUNCTION static + void join( const FunctorType & f , volatile void * const lhs , const volatile void * const rhs ) + { + f.join( ArgTag() , (volatile T *)lhs , (const volatile T *)rhs ); + } +}; + +/* 'join' function provided, no tag, array value */ +template< class FunctorType , class T > +struct FunctorValueJoin + < FunctorType + , void + , T * + // First substitution failure when FunctorType::join does not exist. +#if defined( KOKKOS_HAVE_CXX11 ) + // Second substitution failure when enable_if( & Functor::join ) does not exist + , decltype( FunctorValueJoinFunction< FunctorType , void >::enable_if( & FunctorType::join ) ) +#else + , typename Impl::enable_if< 0 < sizeof( & FunctorType::join ) >::type +#endif + > +{ + KOKKOS_FORCEINLINE_FUNCTION static + void join( const FunctorType & f , volatile void * const lhs , const volatile void * const rhs ) + { + f.join( (volatile T *)lhs , (const volatile T *)rhs ); + } +}; + +} // namespace Impl +} // namespace Kokkos + +#ifdef KOKKOS_HAVE_CXX11 +namespace Kokkos { + +namespace Impl { + + template + struct JoinLambdaAdapter { + typedef ValueType value_type; + const JoinOp& lambda; + KOKKOS_INLINE_FUNCTION + JoinLambdaAdapter(const JoinOp& lambda_):lambda(lambda_) {} + + KOKKOS_INLINE_FUNCTION + void join(volatile value_type& dst, const volatile value_type& src) const { + lambda(dst,src); + } + + KOKKOS_INLINE_FUNCTION + void join(value_type& dst, const value_type& src) const { + lambda(dst,src); + } + + KOKKOS_INLINE_FUNCTION + void operator() (volatile value_type& dst, const volatile value_type& src) const { + lambda(dst,src); + } + + KOKKOS_INLINE_FUNCTION + void operator() (value_type& dst, const value_type& src) const { + lambda(dst,src); + } + }; + + template + struct JoinLambdaAdapter::enable_if( & JoinOp::join ) )> { + typedef ValueType value_type; + typedef StaticAssertSame assert_value_types_match; + const JoinOp& lambda; + KOKKOS_INLINE_FUNCTION + JoinLambdaAdapter(const JoinOp& lambda_):lambda(lambda_) {} + + KOKKOS_INLINE_FUNCTION + void join(volatile value_type& dst, const volatile value_type& src) const { + lambda.join(dst,src); + } + + KOKKOS_INLINE_FUNCTION + void join(value_type& dst, const value_type& src) const { + lambda.join(dst,src); + } + + KOKKOS_INLINE_FUNCTION + void operator() (volatile value_type& dst, const volatile value_type& src) const { + lambda.join(dst,src); + } + + KOKKOS_INLINE_FUNCTION + void operator() (value_type& dst, const value_type& src) const { + lambda.join(dst,src); + } + }; + + template + struct JoinAdd { + typedef ValueType value_type; + + KOKKOS_INLINE_FUNCTION + JoinAdd() {} + + KOKKOS_INLINE_FUNCTION + void join(volatile value_type& dst, const volatile value_type& src) const { + dst+=src; + } + KOKKOS_INLINE_FUNCTION + void operator() (value_type& dst, const value_type& src) const { + dst+=src; + } + KOKKOS_INLINE_FUNCTION + void operator() (volatile value_type& dst, const volatile value_type& src) const { + dst+=src; + } + }; + +} +} +#endif + +//---------------------------------------------------------------------------- +//---------------------------------------------------------------------------- + +namespace Kokkos { +namespace Impl { + +template< class FunctorType , class ArgTag + , class T = typename FunctorValueTraits::reference_type > +struct FunctorValueOps ; + +template< class FunctorType , class ArgTag , class T > +struct FunctorValueOps< FunctorType , ArgTag , T & > +{ + KOKKOS_FORCEINLINE_FUNCTION static + T * pointer( T & r ) { return & r ; } + + KOKKOS_FORCEINLINE_FUNCTION static + T & reference( void * p ) { return *((T*)p); } + + KOKKOS_FORCEINLINE_FUNCTION static + void copy( const FunctorType & , void * const lhs , const void * const rhs ) + { *((T*)lhs) = *((const T*)rhs); } +}; + +/* No 'join' function provided, array of values */ +template< class FunctorType , class ArgTag , class T > +struct FunctorValueOps< FunctorType , ArgTag , T * > +{ + KOKKOS_FORCEINLINE_FUNCTION static + T * pointer( T * p ) { return p ; } + + KOKKOS_FORCEINLINE_FUNCTION static + T * reference( void * p ) { return ((T*)p); } + + KOKKOS_FORCEINLINE_FUNCTION static + void copy( const FunctorType & f , void * const lhs , const void * const rhs ) + { + const int n = FunctorValueTraits::value_count(f); + for ( int i = 0 ; i < n ; ++i ) { ((T*)lhs)[i] = ((const T*)rhs)[i]; } + } +}; + +} // namespace Impl +} // namespace Kokkos + +//---------------------------------------------------------------------------- +//---------------------------------------------------------------------------- + +namespace Kokkos { +namespace Impl { + +// Compatible functions for 'final' function and value_type not an array +template< class FunctorType , class ArgTag , bool IsArray = 0 == FunctorValueTraits::StaticValueSize > +struct FunctorFinalFunction { + + typedef typename FunctorValueTraits::value_type value_type ; + + KOKKOS_INLINE_FUNCTION static void enable_if( void (FunctorType::*)( ArgTag , value_type & ) const ); + KOKKOS_INLINE_FUNCTION static void enable_if( void (FunctorType::*)( ArgTag const & , value_type & ) const ); + KOKKOS_INLINE_FUNCTION static void enable_if( void (FunctorType::*)( ArgTag , value_type & ) ); + KOKKOS_INLINE_FUNCTION static void enable_if( void (FunctorType::*)( ArgTag const & , value_type & ) ); + KOKKOS_INLINE_FUNCTION static void enable_if( void ( *)( ArgTag , value_type & ) ); + KOKKOS_INLINE_FUNCTION static void enable_if( void ( *)( ArgTag const & , value_type & ) ); + + // KOKKOS_INLINE_FUNCTION static void enable_if( void (FunctorType::*)( ArgTag , value_type volatile & ) const ); + // KOKKOS_INLINE_FUNCTION static void enable_if( void (FunctorType::*)( ArgTag const & , value_type volatile & ) const ); + // KOKKOS_INLINE_FUNCTION static void enable_if( void (FunctorType::*)( ArgTag , value_type volatile & ) ); + // KOKKOS_INLINE_FUNCTION static void enable_if( void (FunctorType::*)( ArgTag const & , value_type volatile & ) ); + // KOKKOS_INLINE_FUNCTION static void enable_if( void ( *)( ArgTag , value_type volatile & ) ); + // KOKKOS_INLINE_FUNCTION static void enable_if( void ( *)( ArgTag const & , value_type volatile & ) ); + + KOKKOS_INLINE_FUNCTION static void enable_if( void (FunctorType::*)( ArgTag , value_type const & ) const ); + KOKKOS_INLINE_FUNCTION static void enable_if( void (FunctorType::*)( ArgTag const & , value_type const & ) const ); + KOKKOS_INLINE_FUNCTION static void enable_if( void (FunctorType::*)( ArgTag , value_type const & ) ); + KOKKOS_INLINE_FUNCTION static void enable_if( void (FunctorType::*)( ArgTag const & , value_type const & ) ); + KOKKOS_INLINE_FUNCTION static void enable_if( void ( *)( ArgTag , value_type const & ) ); + KOKKOS_INLINE_FUNCTION static void enable_if( void ( *)( ArgTag const & , value_type const & ) ); + + // KOKKOS_INLINE_FUNCTION static void enable_if( void (FunctorType::*)( ArgTag , value_type const volatile & ) const ); + // KOKKOS_INLINE_FUNCTION static void enable_if( void (FunctorType::*)( ArgTag const & , value_type const volatile & ) const ); + // KOKKOS_INLINE_FUNCTION static void enable_if( void (FunctorType::*)( ArgTag , value_type const volatile & ) ); + // KOKKOS_INLINE_FUNCTION static void enable_if( void (FunctorType::*)( ArgTag const & , value_type const volatile & ) ); + // KOKKOS_INLINE_FUNCTION static void enable_if( void ( *)( ArgTag , value_type const volatile & ) ); + // KOKKOS_INLINE_FUNCTION static void enable_if( void ( *)( ArgTag const & , value_type const volatile & ) ); +}; + +// Compatible functions for 'final' function and value_type is an array +template< class FunctorType , class ArgTag > +struct FunctorFinalFunction< FunctorType , ArgTag , true > { + + typedef typename FunctorValueTraits::value_type value_type ; + + KOKKOS_INLINE_FUNCTION static void enable_if( void (FunctorType::*)( ArgTag , value_type * ) const ); + KOKKOS_INLINE_FUNCTION static void enable_if( void (FunctorType::*)( ArgTag const & , value_type * ) const ); + KOKKOS_INLINE_FUNCTION static void enable_if( void (FunctorType::*)( ArgTag , value_type * ) ); + KOKKOS_INLINE_FUNCTION static void enable_if( void (FunctorType::*)( ArgTag const & , value_type * ) ); + KOKKOS_INLINE_FUNCTION static void enable_if( void ( *)( ArgTag , value_type * ) ); + KOKKOS_INLINE_FUNCTION static void enable_if( void ( *)( ArgTag const & , value_type * ) ); + + // KOKKOS_INLINE_FUNCTION static void enable_if( void (FunctorType::*)( ArgTag , value_type volatile * ) const ); + // KOKKOS_INLINE_FUNCTION static void enable_if( void (FunctorType::*)( ArgTag const & , value_type volatile * ) const ); + // KOKKOS_INLINE_FUNCTION static void enable_if( void (FunctorType::*)( ArgTag , value_type volatile * ) ); + // KOKKOS_INLINE_FUNCTION static void enable_if( void (FunctorType::*)( ArgTag const & , value_type volatile * ) ); + // KOKKOS_INLINE_FUNCTION static void enable_if( void ( *)( ArgTag , value_type volatile * ) ); + // KOKKOS_INLINE_FUNCTION static void enable_if( void ( *)( ArgTag const & , value_type volatile * ) ); + + KOKKOS_INLINE_FUNCTION static void enable_if( void (FunctorType::*)( ArgTag , value_type const * ) const ); + KOKKOS_INLINE_FUNCTION static void enable_if( void (FunctorType::*)( ArgTag const & , value_type const * ) const ); + KOKKOS_INLINE_FUNCTION static void enable_if( void (FunctorType::*)( ArgTag , value_type const * ) ); + KOKKOS_INLINE_FUNCTION static void enable_if( void (FunctorType::*)( ArgTag const & , value_type const * ) ); + KOKKOS_INLINE_FUNCTION static void enable_if( void ( *)( ArgTag , value_type const * ) ); + KOKKOS_INLINE_FUNCTION static void enable_if( void ( *)( ArgTag const & , value_type const * ) ); + + // KOKKOS_INLINE_FUNCTION static void enable_if( void (FunctorType::*)( ArgTag , value_type const volatile * ) const ); + // KOKKOS_INLINE_FUNCTION static void enable_if( void (FunctorType::*)( ArgTag const & , value_type const volatile * ) const ); + // KOKKOS_INLINE_FUNCTION static void enable_if( void (FunctorType::*)( ArgTag , value_type const volatile * ) ); + // KOKKOS_INLINE_FUNCTION static void enable_if( void (FunctorType::*)( ArgTag const & , value_type const volatile * ) ); + // KOKKOS_INLINE_FUNCTION static void enable_if( void ( *)( ArgTag , value_type const volatile * ) ); + // KOKKOS_INLINE_FUNCTION static void enable_if( void ( *)( ArgTag const & , value_type const volatile * ) ); +}; + +template< class FunctorType > +struct FunctorFinalFunction< FunctorType , void , false > { + + typedef typename FunctorValueTraits::value_type value_type ; + + KOKKOS_INLINE_FUNCTION static void enable_if( void (FunctorType::*)( value_type & ) const ); + KOKKOS_INLINE_FUNCTION static void enable_if( void (FunctorType::*)( value_type & ) ); + KOKKOS_INLINE_FUNCTION static void enable_if( void ( *)( value_type & ) ); + + KOKKOS_INLINE_FUNCTION static void enable_if( void (FunctorType::*)( const value_type & ) const ); + KOKKOS_INLINE_FUNCTION static void enable_if( void (FunctorType::*)( const value_type & ) ); + KOKKOS_INLINE_FUNCTION static void enable_if( void ( *)( const value_type & ) ); +}; + +template< class FunctorType > +struct FunctorFinalFunction< FunctorType , void , true > { + + typedef typename FunctorValueTraits::value_type value_type ; + + KOKKOS_INLINE_FUNCTION static void enable_if( void (FunctorType::*)( value_type * ) const ); + KOKKOS_INLINE_FUNCTION static void enable_if( void (FunctorType::*)( value_type * ) ); + KOKKOS_INLINE_FUNCTION static void enable_if( void ( *)( value_type * ) ); + + KOKKOS_INLINE_FUNCTION static void enable_if( void (FunctorType::*)( const value_type * ) const ); + KOKKOS_INLINE_FUNCTION static void enable_if( void (FunctorType::*)( const value_type * ) ); + KOKKOS_INLINE_FUNCTION static void enable_if( void ( *)( const value_type * ) ); +}; + +/* No 'final' function provided */ +template< class FunctorType , class ArgTag + , class ResultType = typename FunctorValueTraits::reference_type + , class Enable = void > +struct FunctorFinal +{ + KOKKOS_FORCEINLINE_FUNCTION static + void final( const FunctorType & , void * ) {} +}; + +/* 'final' function provided */ +template< class FunctorType , class ArgTag , class T > +struct FunctorFinal + < FunctorType + , ArgTag + , T & + // First substitution failure when FunctorType::final does not exist. +#if defined( KOKKOS_HAVE_CXX11 ) + // Second substitution failure when enable_if( & Functor::final ) does not exist + , decltype( FunctorFinalFunction< FunctorType , ArgTag >::enable_if( & FunctorType::final ) ) +#else + , typename Impl::enable_if< 0 < sizeof( & FunctorType::final ) >::type +#endif + > +{ + KOKKOS_FORCEINLINE_FUNCTION static + void final( const FunctorType & f , void * p ) { f.final( *((T*)p) ); } + + KOKKOS_FORCEINLINE_FUNCTION static + void final( FunctorType & f , void * p ) { f.final( *((T*)p) ); } +}; + +/* 'final' function provided for array value */ +template< class FunctorType , class ArgTag , class T > +struct FunctorFinal + < FunctorType + , ArgTag + , T * + // First substitution failure when FunctorType::final does not exist. +#if defined( KOKKOS_HAVE_CXX11 ) + // Second substitution failure when enable_if( & Functor::final ) does not exist + , decltype( FunctorFinalFunction< FunctorType , ArgTag >::enable_if( & FunctorType::final ) ) +#else + , typename Impl::enable_if< 0 < sizeof( & FunctorType::final ) >::type +#endif + > +{ + KOKKOS_FORCEINLINE_FUNCTION static + void final( const FunctorType & f , void * p ) { f.final( (T*)p ); } + + KOKKOS_FORCEINLINE_FUNCTION static + void final( FunctorType & f , void * p ) { f.final( (T*)p ); } +}; + +} // namespace Impl +} // namespace Kokkos + +//---------------------------------------------------------------------------- +//---------------------------------------------------------------------------- + +namespace Kokkos { +namespace Impl { + +template< class FunctorType , class ArgTag + , class ReferenceType = typename FunctorValueTraits::reference_type > +struct FunctorApplyFunction { + + KOKKOS_INLINE_FUNCTION static void enable_if( void (FunctorType::*)( ArgTag , ReferenceType ) const ); + KOKKOS_INLINE_FUNCTION static void enable_if( void (FunctorType::*)( ArgTag const & , ReferenceType ) const ); + KOKKOS_INLINE_FUNCTION static void enable_if( void (FunctorType::*)( ArgTag , ReferenceType ) ); + KOKKOS_INLINE_FUNCTION static void enable_if( void (FunctorType::*)( ArgTag const & , ReferenceType ) ); + KOKKOS_INLINE_FUNCTION static void enable_if( void ( *)( ArgTag , ReferenceType ) ); + KOKKOS_INLINE_FUNCTION static void enable_if( void ( *)( ArgTag const & , ReferenceType ) ); +}; + +template< class FunctorType , class ReferenceType > +struct FunctorApplyFunction< FunctorType , void , ReferenceType > { + + KOKKOS_INLINE_FUNCTION static void enable_if( void (FunctorType::*)( ReferenceType ) const ); + KOKKOS_INLINE_FUNCTION static void enable_if( void (FunctorType::*)( ReferenceType ) ); + KOKKOS_INLINE_FUNCTION static void enable_if( void ( *)( ReferenceType ) ); +}; + +template< class FunctorType > +struct FunctorApplyFunction< FunctorType , void , void > { + + KOKKOS_INLINE_FUNCTION static void enable_if( void (FunctorType::*)() const ); + KOKKOS_INLINE_FUNCTION static void enable_if( void (FunctorType::*)() ); +}; + +template< class FunctorType , class ArgTag , class ReferenceType + , class Enable = void > +struct FunctorApply +{ + KOKKOS_FORCEINLINE_FUNCTION static + void apply( const FunctorType & , void * ) {} +}; + +/* 'apply' function provided for void value */ +template< class FunctorType , class ArgTag > +struct FunctorApply + < FunctorType + , ArgTag + , void + // First substitution failure when FunctorType::apply does not exist. +#if defined( KOKKOS_HAVE_CXX11 ) + // Second substitution failure when enable_if( & Functor::apply ) does not exist + , decltype( FunctorApplyFunction< FunctorType , ArgTag , void >::enable_if( & FunctorType::apply ) ) +#else + , typename Impl::enable_if< 0 < sizeof( & FunctorType::apply ) >::type +#endif + > +{ + KOKKOS_FORCEINLINE_FUNCTION static + void apply( FunctorType & f ) { f.apply(); } + + KOKKOS_FORCEINLINE_FUNCTION static + void apply( const FunctorType & f ) { f.apply(); } +}; + +/* 'apply' function provided for single value */ +template< class FunctorType , class ArgTag , class T > +struct FunctorApply + < FunctorType + , ArgTag + , T & + // First substitution failure when FunctorType::apply does not exist. +#if defined( KOKKOS_HAVE_CXX11 ) + // Second substitution failure when enable_if( & Functor::apply ) does not exist + , decltype( FunctorApplyFunction< FunctorType , ArgTag >::enable_if( & FunctorType::apply ) ) +#else + , typename Impl::enable_if< 0 < sizeof( & FunctorType::apply ) >::type +#endif + > +{ + KOKKOS_FORCEINLINE_FUNCTION static + void apply( const FunctorType & f , void * p ) { f.apply( *((T*)p) ); } + + KOKKOS_FORCEINLINE_FUNCTION static + void apply( FunctorType & f , void * p ) { f.apply( *((T*)p) ); } +}; + +} // namespace Impl +} // namespace Kokkos + +//---------------------------------------------------------------------------- +//---------------------------------------------------------------------------- + +#endif /* KOKKOS_FUNCTORADAPTER_HPP */ + diff --git a/lib/kokkos/core/src/impl/Kokkos_HostSpace.cpp b/lib/kokkos/core/src/impl/Kokkos_HostSpace.cpp new file mode 100755 index 0000000000..ecb779a4c8 --- /dev/null +++ b/lib/kokkos/core/src/impl/Kokkos_HostSpace.cpp @@ -0,0 +1,271 @@ +/* +//@HEADER +// ************************************************************************ +// +// Kokkos: Manycore Performance-Portable Multidimensional Arrays +// Copyright (2012) Sandia Corporation +// +// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +// the U.S. Government retains certain rights in this software. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// 1. Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// 3. Neither the name of the Corporation nor the names of the +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Questions? Contact H. Carter Edwards (hcedwar@sandia.gov) +// +// ************************************************************************ +//@HEADER +*/ + +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +/*--------------------------------------------------------------------------*/ + +namespace Kokkos { +namespace Impl { +namespace { + +Impl::MemoryTracking<> & host_space_singleton() +{ + static Impl::MemoryTracking<> self("Kokkos::HostSpace"); + return self ; +} + +} // namespace +} // namespace Impl +} // namespade Kokkos + +/*--------------------------------------------------------------------------*/ + +namespace Kokkos { +namespace Impl { + +void * host_allocate_not_thread_safe( const std::string & label , const size_t size ) +{ + void * ptr = 0 ; + + if ( size ) { + size_t size_padded = size ; + void * ptr_alloc = 0 ; + +#if defined( __INTEL_COMPILER ) && !defined ( KOKKOS_HAVE_CUDA ) + + ptr = ptr_alloc = _mm_malloc( size , MEMORY_ALIGNMENT ); + +#elif ( defined( _POSIX_C_SOURCE ) && _POSIX_C_SOURCE >= 200112L ) || \ + ( defined( _XOPEN_SOURCE ) && _XOPEN_SOURCE >= 600 ) + + posix_memalign( & ptr_alloc , MEMORY_ALIGNMENT , size ); + ptr = ptr_alloc ; + +#else + + { + // Over-allocate to and round up to guarantee proper alignment. + + size_padded = ( size + MEMORY_ALIGNMENT - 1 ); + + ptr_alloc = malloc( size_padded ); + + const size_t rem = reinterpret_cast(ptr_alloc) % MEMORY_ALIGNMENT ; + + ptr = static_cast(ptr_alloc) + ( rem ? MEMORY_ALIGNMENT - rem : 0 ); + } + +#endif + + if ( ptr_alloc && ptr_alloc <= ptr && + 0 == ( reinterpret_cast(ptr) % MEMORY_ALIGNMENT ) ) { + // Insert allocated pointer and allocation count + Impl::host_space_singleton().insert( label , ptr_alloc , size_padded ); + } + else { + std::ostringstream msg ; + msg << "Kokkos::Impl::host_allocate_not_thread_safe( " + << label + << " , " << size + << " ) FAILED aligned memory allocation" ; + Kokkos::Impl::throw_runtime_exception( msg.str() ); + } + } + + return ptr ; +} + +void host_decrement_not_thread_safe( const void * ptr ) +{ + void * ptr_alloc = Impl::host_space_singleton().decrement( ptr ); + + if ( ptr_alloc ) { +#if defined( __INTEL_COMPILER ) && !defined ( KOKKOS_HAVE_CUDA ) + _mm_free( ptr_alloc ); +#else + free( ptr_alloc ); +#endif + } +} + +DeepCopy::DeepCopy( void * dst , const void * src , size_t n ) +{ + memcpy( dst , src , n ); +} + +} +} + +//---------------------------------------------------------------------------- +//---------------------------------------------------------------------------- + +namespace Kokkos { +namespace { + +static const int QUERY_SPACE_IN_PARALLEL_MAX = 16 ; + +typedef int (* QuerySpaceInParallelPtr )(); + +QuerySpaceInParallelPtr s_in_parallel_query[ QUERY_SPACE_IN_PARALLEL_MAX ] ; +int s_in_parallel_query_count = 0 ; + +} // namespace + +void HostSpace::register_in_parallel( int (*device_in_parallel)() ) +{ + if ( 0 == device_in_parallel ) { + Kokkos::Impl::throw_runtime_exception( std::string("Kokkos::HostSpace::register_in_parallel ERROR : given NULL" ) ); + } + + int i = -1 ; + + if ( ! (device_in_parallel)() ) { + for ( i = 0 ; i < s_in_parallel_query_count && ! (*(s_in_parallel_query[i]))() ; ++i ); + } + + if ( i < s_in_parallel_query_count ) { + Kokkos::Impl::throw_runtime_exception( std::string("Kokkos::HostSpace::register_in_parallel_query ERROR : called in_parallel" ) ); + + } + + if ( QUERY_SPACE_IN_PARALLEL_MAX <= i ) { + Kokkos::Impl::throw_runtime_exception( std::string("Kokkos::HostSpace::register_in_parallel_query ERROR : exceeded maximum" ) ); + + } + + for ( i = 0 ; i < s_in_parallel_query_count && s_in_parallel_query[i] != device_in_parallel ; ++i ); + + if ( i == s_in_parallel_query_count ) { + s_in_parallel_query[s_in_parallel_query_count++] = device_in_parallel ; + } +} + +int HostSpace::in_parallel() +{ + const int n = s_in_parallel_query_count ; + + int i = 0 ; + + while ( i < n && ! (*(s_in_parallel_query[i]))() ) { ++i ; } + + return i < n ; +} + +} // namespace Kokkos + +/*--------------------------------------------------------------------------*/ + +namespace Kokkos { + +void * HostSpace::allocate( const std::string & label , const size_t size ) +{ + void * ptr = 0 ; + + if ( ! HostSpace::in_parallel() ) { + ptr = Impl::host_allocate_not_thread_safe( label , size ); + } + else { + Kokkos::Impl::throw_runtime_exception( std::string("Kokkos::HostSpace::allocate called within a parallel functor") ); + } + + return ptr ; +} + +void HostSpace::increment( const void * ptr ) +{ + if ( ! HostSpace::in_parallel() ) { + Impl::host_space_singleton().increment( ptr ); + } + else { + Kokkos::Impl::throw_runtime_exception( std::string("Kokkos::HostSpace::increment called within a parallel functor") ); + } +} + +void HostSpace::decrement( const void * ptr ) +{ + if ( ! HostSpace::in_parallel() ) { + Impl::host_decrement_not_thread_safe( ptr ); + } + else { + Kokkos::Impl::throw_runtime_exception( std::string("Kokkos::HostSpace::decrement called within a parallel functor") ); + } +} + +int HostSpace::count( const void * ptr ) { + if ( ! HostSpace::in_parallel() ) { + Impl::MemoryTracking<>::Entry * const entry = + Impl::host_space_singleton().query(ptr); + return entry != NULL?entry->count():0; + } + else { + Kokkos::Impl::throw_runtime_exception( std::string("Kokkos::HostSpace::count called within a parallel functor") ); + return -1; + } +} + +void HostSpace::print_memory_view( std::ostream & o ) +{ + Impl::host_space_singleton().print( o , std::string(" ") ); +} + +std::string HostSpace::query_label( const void * p ) +{ + Impl::MemoryTracking<>::Entry * const entry = Impl::host_space_singleton().query(p); + return std::string( entry ? entry->label() : "" ); +} + +} // namespace Kokkos + +/*--------------------------------------------------------------------------*/ +/*--------------------------------------------------------------------------*/ + diff --git a/lib/kokkos/core/src/impl/Kokkos_MemoryTracking.hpp b/lib/kokkos/core/src/impl/Kokkos_MemoryTracking.hpp new file mode 100755 index 0000000000..3883fc130a --- /dev/null +++ b/lib/kokkos/core/src/impl/Kokkos_MemoryTracking.hpp @@ -0,0 +1,374 @@ +/* +//@HEADER +// ************************************************************************ +// +// Kokkos: Manycore Performance-Portable Multidimensional Arrays +// Copyright (2012) Sandia Corporation +// +// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +// the U.S. Government retains certain rights in this software. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// 1. Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// 3. Neither the name of the Corporation nor the names of the +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Questions? Contact H. Carter Edwards (hcedwar@sandia.gov) +// +// ************************************************************************ +//@HEADER +*/ + +#ifndef KOKKOS_MEMORY_TRACKING_HPP +#define KOKKOS_MEMORY_TRACKING_HPP + +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +namespace Kokkos { +namespace Impl { +namespace { + +// Fast search for result[-1] <= val < result[0]. +// Requires result[max] == upper_bound. +// Start with a binary search until the search range is +// less than LINEAR_LIMIT, then switch to linear search. + +int memory_tracking_upper_bound( const ptrdiff_t * const begin + , unsigned length + , const ptrdiff_t value ) +{ + enum { LINEAR_LIMIT = 32 }; + + // precondition: begin[length-1] == std::numeric_limits::max() + + const ptrdiff_t * first = begin ; + + while ( LINEAR_LIMIT < length ) { + unsigned half = length >> 1 ; + const ptrdiff_t * middle = first + half ; + + if ( value < *middle ) { + length = half ; + } + else { + first = ++middle ; + length -= ++half ; + } + } + + for ( ; ! ( value < *first ) ; ++first ) {} + + return first - begin ; +} + +template< class AttributeType = size_t > +class MemoryTracking { +public: + + class Entry { + private: + + friend class MemoryTracking ; + + enum { LABEL_LENGTH = 128 }; + + Entry( const Entry & ); + Entry & operator = ( const Entry & ); + + ~Entry() {} + + Entry() + : m_count(0) + , m_alloc_ptr( reinterpret_cast( std::numeric_limits::max() ) ) + , m_alloc_size(0) + , m_attribute() + { strcpy( m_label , "sentinel" ); } + + Entry( const std::string & arg_label + , void * const arg_alloc_ptr + , size_t const arg_alloc_size ) + : m_count( 0 ) + , m_alloc_ptr( arg_alloc_ptr ) + , m_alloc_size( arg_alloc_size ) + , m_attribute() + { + strncpy( m_label , arg_label.c_str() , LABEL_LENGTH ); + m_label[ LABEL_LENGTH - 1 ] = 0 ; + } + + char m_label[ LABEL_LENGTH ] ; + size_t m_count ; + + public: + + void * const m_alloc_ptr ; + size_t const m_alloc_size ; + AttributeType m_attribute ; + + size_t count() const { return m_count ; } + const char * label() const { return m_label ; } + + void print( std::ostream & oss ) const + { + oss << "{ \"" << m_label + << "\" count(" << m_count + << ") memory[ " << m_alloc_ptr + << " + " << m_alloc_size + << " ]" ; + } + }; + + //------------------------------------------------------------ + /** \brief Track a memory range defined by the entry. + * Return the input entry pointer for success. + * Throw exception for failure. + */ + Entry * insert( const std::string & arg_label + , void * const arg_alloc_ptr + , size_t const arg_alloc_size + ) + { + Entry * result = 0 ; + + const ptrdiff_t alloc_begin = reinterpret_cast(arg_alloc_ptr); + const ptrdiff_t alloc_end = alloc_begin + arg_alloc_size ; + + const bool ok_exist = ! m_tracking_end.empty(); + + const bool ok_input = + ok_exist && + ( 0 < alloc_begin ) && + ( alloc_begin < alloc_end ) && + ( alloc_end < std::numeric_limits::max() ); + + const int i = ok_input + ? memory_tracking_upper_bound( & m_tracking_end[0] , m_tracking_end.size() , alloc_end ) + : -1 ; + + const bool ok_range = ( 0 <= i ) && ( alloc_end <= reinterpret_cast( m_tracking[i]->m_alloc_ptr ) ); + + // allocate the new entry only if the vector inserts succeed. + const bool ok_insert = + ok_range && + ( alloc_end == *m_tracking_end.insert(m_tracking_end.begin()+i,alloc_end) ) && + ( 0 == *m_tracking.insert(m_tracking.begin()+i,0) ) && + ( 0 != ( result = new Entry(arg_label,arg_alloc_ptr,arg_alloc_size) ) ); + + if ( ok_insert ) { + result->m_count = 1 ; + m_tracking[i] = result ; + } + else { + std::ostringstream msg ; + msg << m_space + << "::insert( " << arg_label + << " , " << arg_alloc_ptr + << " , " << arg_alloc_size + << " ) ERROR : " ; + if ( ! ok_exist ) { + msg << " called after return from main()" ; + } + else if ( ! ok_input ) { + msg << " bad allocation range" ; + } + else if ( ! ok_range ) { + msg << " overlapping memory range with" + << " { " << m_tracking[i]->m_label + << " , " << m_tracking[i]->m_alloc_ptr + << " , " << m_tracking[i]->m_alloc_size + << " }" ; + } + else { + msg << " internal allocation error" ; + } + Kokkos::Impl::throw_runtime_exception( msg.str() ); + } + + return result ; + } + + /** \brief Decrement the tracked memory range. + * If the count is zero then return the originally inserted pointer. + * If the count is non zero then return zero. + */ + void * decrement( void const * const ptr ) + { + void * result = 0 ; + + if ( ptr ) { + const bool ok_exist = ! m_tracking_end.empty(); + + const int i = ok_exist + ? memory_tracking_upper_bound( & m_tracking_end[0] , m_tracking_end.size() , reinterpret_cast(ptr) ) + : -1 ; + + const bool ok_found = ( 0 <= i ) && ( reinterpret_cast( m_tracking[i]->m_alloc_ptr ) <= + reinterpret_cast(ptr) ); + + if ( ok_found ) { + if ( 0 == --( m_tracking[i]->m_count ) ) { + result = m_tracking[i]->m_alloc_ptr ; + delete m_tracking[i] ; + m_tracking.erase( m_tracking.begin() + i ); + m_tracking_end.erase( m_tracking_end.begin() + i ); + } + } + else { + // Don't throw as this is likely called from within a destructor. + std::cerr << m_space + << "::decrement( " << ptr << " ) ERROR : " + << ( ! ok_exist ? " called after return from main()" + : " memory not being tracked" ) + << std::endl ; + std::cerr.flush(); + } + } + return result ; + } + + /** \brief Increment the tracking count. */ + void increment( void const * const ptr ) + { + if ( ptr ) { + const bool ok_exist = ! m_tracking_end.empty(); + + const int i = ok_exist + ? memory_tracking_upper_bound( & m_tracking_end[0] , m_tracking_end.size() , reinterpret_cast(ptr) ) + : -1 ; + + const bool ok_found = ( 0 <= i ) && ( reinterpret_cast( m_tracking[i]->m_alloc_ptr ) <= + reinterpret_cast(ptr) ); + + if ( ok_found ) { + ++( m_tracking[i]->m_count ); + } + else { + std::ostringstream msg ; + msg << m_space + << "::increment( " << ptr << " ) ERROR : " + << ( ! ok_exist ? " called after return from main()" + : " memory not being tracked" ) + << std::endl ; + Kokkos::Impl::throw_runtime_exception( msg.str() ); + } + } + } + + /** \brief Query a tracked memory range. + * Return zero for not found. + */ + Entry * query( void const * const ptr ) const + { + const bool ok_exist = ! m_tracking_end.empty(); + + const int i = ( ok_exist && ptr ) + ? memory_tracking_upper_bound( & m_tracking_end[0] , m_tracking_end.size() , reinterpret_cast(ptr) ) + : -1 ; + + const bool ok_found = ( 0 <= i ) && ( reinterpret_cast( m_tracking[i]->m_alloc_ptr ) <= + reinterpret_cast(ptr) ); + + return ok_found ? m_tracking[i] : (Entry *) 0 ; + } + + /** \brief Call the 'print' method on all entries. */ + void print( std::ostream & oss , const std::string & lead ) const + { + const size_t n = m_tracking.empty() ? 0 : m_tracking.size() - 1 ; + for ( size_t i = 0 ; i < n ; ++i ) { + oss << lead ; + m_tracking[i]->print( oss ); + oss << std::endl ; + } + } + + size_t size() const { return m_tracking.size(); } + + template< typename iType > + MemoryTracking & operator[]( const iType & i ) const + { return *m_tracking[i]; } + + /** \brief Construct with a name for error messages */ + explicit MemoryTracking( const std::string & space_name ) + : m_space( space_name ) + , m_tracking() + , m_tracking_end() + , m_sentinel() + { + m_tracking.reserve( 512 ); + m_tracking_end.reserve( 512 ); + m_tracking.push_back( & m_sentinel ); + m_tracking_end.push_back( reinterpret_cast( m_sentinel.m_alloc_ptr ) ); + } + + /** \brief Print memory leak warning for all entries. */ + ~MemoryTracking() + { + try { + const ptrdiff_t max = std::numeric_limits::max(); + + if ( 1 < m_tracking.size() ) { + std::cerr << m_space << " destroyed with memory leaks:" ; + print( std::cerr , std::string(" ") ); + } + else if ( m_tracking.empty() || max != m_tracking_end.back() ) { + std::cerr << m_space << " corrupted data structure" << std::endl ; + } + + m_space = std::string(); + m_tracking = std::vector(); + m_tracking_end = std::vector(); + } + catch( ... ) {} + } + + const std::string & label() const { return m_space ; } + +private: + MemoryTracking(); + MemoryTracking( const MemoryTracking & ); + MemoryTracking & operator = ( const MemoryTracking & ); + + std::string m_space ; + std::vector m_tracking ; + std::vector m_tracking_end ; + Entry m_sentinel ; +}; + +} /* namespace */ +} /* namespace Impl */ +} /* namespace Kokkos */ + +#endif + diff --git a/lib/kokkos/core/src/impl/Kokkos_Memory_Fence.hpp b/lib/kokkos/core/src/impl/Kokkos_Memory_Fence.hpp new file mode 100755 index 0000000000..eebb0c7f01 --- /dev/null +++ b/lib/kokkos/core/src/impl/Kokkos_Memory_Fence.hpp @@ -0,0 +1,73 @@ +/* +//@HEADER +// ************************************************************************ +// +// Kokkos: Manycore Performance-Portable Multidimensional Arrays +// Copyright (2012) Sandia Corporation +// +// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +// the U.S. Government retains certain rights in this software. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// 1. Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// 3. Neither the name of the Corporation nor the names of the +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Questions? Contact H. Carter Edwards (hcedwar@sandia.gov) +// +// ************************************************************************ +//@HEADER +*/ + +#if defined( KOKKOS_ATOMIC_HPP ) && ! defined( KOKKOS_MEMORY_FENCE ) +#define KOKKOS_MEMORY_FENCE + +namespace Kokkos { + +//---------------------------------------------------------------------------- + +KOKKOS_FORCEINLINE_FUNCTION +void memory_fence() +{ +#if defined( KOKKOS_ATOMICS_USE_CUDA ) + __threadfence(); +#elif defined( KOKKOS_ATOMICS_USE_GCC ) || \ + ( defined( KOKKOS_COMPILER_NVCC ) && defined( KOKKOS_ATOMICS_USE_INTEL ) ) + __sync_synchronize(); +#elif defined( KOKKOS_ATOMICS_USE_INTEL ) + _mm_mfence(); +#elif defined( KOKKOS_ATOMICS_USE_OMP31 ) + #pragma omp flush + +#else + #error "Error: memory_fence() not defined" +#endif +} + +} // namespace kokkos + +#endif + + diff --git a/lib/kokkos/core/src/impl/Kokkos_PhysicalLayout.hpp b/lib/kokkos/core/src/impl/Kokkos_PhysicalLayout.hpp new file mode 100755 index 0000000000..0dcb3977a3 --- /dev/null +++ b/lib/kokkos/core/src/impl/Kokkos_PhysicalLayout.hpp @@ -0,0 +1,84 @@ +/* +//@HEADER +// ************************************************************************ +// +// Kokkos: Manycore Performance-Portable Multidimensional Arrays +// Copyright (2012) Sandia Corporation +// +// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +// the U.S. Government retains certain rights in this software. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// 1. Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// 3. Neither the name of the Corporation nor the names of the +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Questions? Contact H. Carter Edwards (hcedwar@sandia.gov) +// +// ************************************************************************ +//@HEADER +*/ + +#ifndef KOKKOS_PHYSICAL_LAYOUT_HPP +#define KOKKOS_PHYSICAL_LAYOUT_HPP + + +#include +namespace Kokkos { +namespace Impl { + + + +struct PhysicalLayout { + enum LayoutType {Left,Right,Scalar,Error}; + LayoutType layout_type; + int rank; + long long int stride[8]; //distance between two neighboring elements in a given dimension + + template< class T , class L , class D , class M > + PhysicalLayout( const View & view ) + : layout_type( is_same< typename View::array_layout , LayoutLeft >::value ? Left : ( + is_same< typename View::array_layout , LayoutRight >::value ? Right : Error )) + , rank( view.Rank ) + { + for(int i=0;i<8;i++) stride[i] = 0; + view.stride( stride ); + } + #ifdef KOKKOS_HAVE_CUDA + template< class T , class L , class D , class M > + PhysicalLayout( const View & view ) + : layout_type( is_same< typename View::array_layout , LayoutLeft >::value ? Left : ( + is_same< typename View::array_layout , LayoutRight >::value ? Right : Error )) + , rank( view.Rank ) + { + for(int i=0;i<8;i++) stride[i] = 0; + view.stride( stride ); + } + #endif +}; + +} +} +#endif diff --git a/lib/kokkos/core/src/impl/Kokkos_Serial.cpp b/lib/kokkos/core/src/impl/Kokkos_Serial.cpp new file mode 100755 index 0000000000..db9f7c5b5c --- /dev/null +++ b/lib/kokkos/core/src/impl/Kokkos_Serial.cpp @@ -0,0 +1,119 @@ +/* +//@HEADER +// ************************************************************************ +// +// Kokkos: Manycore Performance-Portable Multidimensional Arrays +// Copyright (2012) Sandia Corporation +// +// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +// the U.S. Government retains certain rights in this software. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// 1. Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// 3. Neither the name of the Corporation nor the names of the +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Questions? Contact H. Carter Edwards (hcedwar@sandia.gov) +// +// ************************************************************************ +//@HEADER +*/ + +#include +#include +#include +#include +#include + +#if defined( KOKKOS_HAVE_SERIAL ) + +/*--------------------------------------------------------------------------*/ + +namespace Kokkos { +namespace Impl { +namespace SerialImpl { + +Sentinel::Sentinel() : m_scratch(0), m_reduce_end(0), m_shared_end(0) {} + +Sentinel::~Sentinel() +{ + if ( m_scratch ) { free( m_scratch ); } + m_scratch = 0 ; + m_reduce_end = 0 ; + m_shared_end = 0 ; +} + +Sentinel & Sentinel::singleton() +{ + static Sentinel s ; return s ; +} + +inline +unsigned align( unsigned n ) +{ + enum { ALIGN = 0x0100 /* 256 */ , MASK = ALIGN - 1 }; + return ( n + MASK ) & ~MASK ; +} + +} // namespace + +SerialTeamMember::SerialTeamMember( int arg_league_rank + , int arg_league_size + , int arg_shared_size + ) + : m_space( ((char *) SerialImpl::Sentinel::singleton().m_scratch) + SerialImpl::Sentinel::singleton().m_reduce_end + , arg_shared_size ) + , m_league_rank( arg_league_rank ) + , m_league_size( arg_league_size ) +{} + +} // namespace Impl + +void * Serial::scratch_memory_resize( unsigned reduce_size , unsigned shared_size ) +{ + static Impl::SerialImpl::Sentinel & s = Impl::SerialImpl::Sentinel::singleton(); + + reduce_size = Impl::SerialImpl::align( reduce_size ); + shared_size = Impl::SerialImpl::align( shared_size ); + + if ( ( s.m_reduce_end < reduce_size ) || + ( s.m_shared_end < s.m_reduce_end + shared_size ) ) { + + if ( s.m_scratch ) { free( s.m_scratch ); } + + if ( s.m_reduce_end < reduce_size ) s.m_reduce_end = reduce_size ; + if ( s.m_shared_end < s.m_reduce_end + shared_size ) s.m_shared_end = s.m_reduce_end + shared_size ; + + s.m_scratch = malloc( s.m_shared_end ); + } + + return s.m_scratch ; +} + +} // namespace Kokkos + +#endif // defined( KOKKOS_HAVE_SERIAL ) + + diff --git a/lib/kokkos/core/src/impl/Kokkos_Serial_TaskPolicy.cpp b/lib/kokkos/core/src/impl/Kokkos_Serial_TaskPolicy.cpp new file mode 100755 index 0000000000..d814a78dfb --- /dev/null +++ b/lib/kokkos/core/src/impl/Kokkos_Serial_TaskPolicy.cpp @@ -0,0 +1,324 @@ +/* +//@HEADER +// ************************************************************************ +// +// Kokkos: Manycore Performance-Portable Multidimensional Arrays +// Copyright (2012) Sandia Corporation +// +// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +// the U.S. Government retains certain rights in this software. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// 1. Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// 3. Neither the name of the Corporation nor the names of the +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Questions? Contact H. Carter Edwards (hcedwar@sandia.gov) +// +// ************************************************************************ +//@HEADER +*/ + +// Experimental unified task-data parallel manycore LDRD + +#include + +#if defined( KOKKOS_HAVE_SERIAL ) +#include +#include +#include +#include +#include + +//---------------------------------------------------------------------------- + +namespace Kokkos { +namespace Impl { + +typedef TaskMember< Kokkos::Serial , void , void > Task ; + +//---------------------------------------------------------------------------- + +namespace { + +inline +unsigned padded_sizeof_derived( unsigned sizeof_derived ) +{ + return sizeof_derived + + ( sizeof_derived % sizeof(Task*) ? sizeof(Task*) - sizeof_derived % sizeof(Task*) : 0 ); +} + +} // namespace + +void Task::deallocate( void * ptr ) +{ + free( ptr ); +} + +void * Task::allocate( const unsigned arg_sizeof_derived + , const unsigned arg_dependence_capacity ) +{ + return malloc( padded_sizeof_derived( arg_sizeof_derived ) + arg_dependence_capacity * sizeof(Task*) ); +} + +Task::~TaskMember() +{ + +} + +Task::TaskMember( const Task::function_verify_type arg_verify + , const Task::function_dealloc_type arg_dealloc + , const Task::function_apply_type arg_apply + , const unsigned arg_sizeof_derived + , const unsigned arg_dependence_capacity + ) + : m_dealloc( arg_dealloc ) + , m_verify( arg_verify ) + , m_apply( arg_apply ) + , m_dep( (Task **)( ((unsigned char *) this) + padded_sizeof_derived( arg_sizeof_derived ) ) ) + , m_wait( 0 ) + , m_next( 0 ) + , m_dep_capacity( arg_dependence_capacity ) + , m_dep_size( 0 ) + , m_ref_count( 0 ) + , m_state( TASK_STATE_CONSTRUCTING ) +{ + for ( unsigned i = 0 ; i < arg_dependence_capacity ; ++i ) m_dep[i] = 0 ; +} + +Task::TaskMember( const Task::function_dealloc_type arg_dealloc + , const Task::function_apply_type arg_apply + , const unsigned arg_sizeof_derived + , const unsigned arg_dependence_capacity + ) + : m_dealloc( arg_dealloc ) + , m_verify( & Task::verify_type ) + , m_apply( arg_apply ) + , m_dep( (Task **)( ((unsigned char *) this) + padded_sizeof_derived( arg_sizeof_derived ) ) ) + , m_wait( 0 ) + , m_next( 0 ) + , m_dep_capacity( arg_dependence_capacity ) + , m_dep_size( 0 ) + , m_ref_count( 0 ) + , m_state( TASK_STATE_CONSTRUCTING ) +{ + for ( unsigned i = 0 ; i < arg_dependence_capacity ; ++i ) m_dep[i] = 0 ; +} + +//---------------------------------------------------------------------------- + +void Task::throw_error_add_dependence() const +{ + std::cerr << "TaskMember< Serial >::add_dependence ERROR" + << " state(" << m_state << ")" + << " dep_size(" << m_dep_size << ")" + << std::endl ; + throw std::runtime_error("TaskMember< Serial >::add_dependence ERROR"); +} + +void Task::throw_error_verify_type() +{ + throw std::runtime_error("TaskMember< Serial >::verify_type ERROR"); +} + +//---------------------------------------------------------------------------- + +#if defined( KOKKOS_ACTIVE_EXECUTION_MEMORY_SPACE_HOST ) + +void Task::assign( Task ** const lhs , Task * rhs , const bool no_throw ) +{ + static const char msg_error_header[] = "Kokkos::Impl::TaskManager::assign ERROR" ; + static const char msg_error_count[] = ": negative reference count" ; + static const char msg_error_complete[] = ": destroy task that is not complete" ; + static const char msg_error_dependences[] = ": destroy task that has dependences" ; + static const char msg_error_exception[] = ": caught internal exception" ; + + const char * msg_error = 0 ; + + try { + + if ( *lhs ) { + + const int count = --((**lhs).m_ref_count); + + if ( 0 == count ) { + + // Reference count at zero, delete it + + // Should only be deallocating a completed task + if ( (**lhs).m_state == Kokkos::TASK_STATE_COMPLETE ) { + + // A completed task should not have dependences... + for ( int i = 0 ; i < (**lhs).m_dep_size && 0 == msg_error ; ++i ) { + if ( (**lhs).m_dep[i] ) msg_error = msg_error_dependences ; + } + } + else { + msg_error = msg_error_complete ; + } + + if ( 0 == msg_error ) { + // Get deletion function and apply it + const Task::function_dealloc_type d = (**lhs).m_dealloc ; + + (*d)( *lhs ); + } + } + else if ( count <= 0 ) { + msg_error = msg_error_count ; + } + } + + if ( 0 == msg_error && rhs ) { ++( rhs->m_ref_count ); } + + *lhs = rhs ; + } + catch( ... ) { + if ( 0 == msg_error ) msg_error = msg_error_exception ; + } + + if ( 0 != msg_error ) { + if ( no_throw ) { + std::cerr << msg_error_header << msg_error << std::endl ; + std::cerr.flush(); + } + else { + std::string msg(msg_error_header); + msg.append(msg_error); + throw std::runtime_error( msg ); + } + } +} +#endif + +namespace { + +Task * s_ready = 0 ; +Task * s_denied = reinterpret_cast( ~((unsigned long)0) ); + +} + +void Task::schedule() +{ + // Execute ready tasks in case the task being scheduled + // is dependent upon a waiting and ready task. + + Task::execute_ready_tasks(); + + // spawning : Constructing -> Waiting + // respawning : Executing -> Waiting + // updating : Waiting -> Waiting + + // Must not be in a dependence linked list: 0 == t->m_next + + const bool ok_state = TASK_STATE_COMPLETE != m_state ; + const bool ok_list = 0 == m_next ; + + if ( ok_state && ok_list ) { + + // Will be waiting for execution upon return from this function + + m_state = Kokkos::TASK_STATE_WAITING ; + + // Insert this task into another dependence that is not complete + + int i = 0 ; + for ( ; i < m_dep_size ; ++i ) { + Task * const y = m_dep[i] ; + if ( y && s_denied != ( m_next = y->m_wait ) ) { + y->m_wait = this ; // CAS( & y->m_wait , m_next , this ); + break ; + } + } + if ( i == m_dep_size ) { + // All dependences are complete, insert into the ready list + m_next = s_ready ; + s_ready = this ; // CAS( & s_ready , m_next = s_ready , this ); + } + } + else { + throw std::runtime_error(std::string("Kokkos::Impl::Task spawn or respawn state error")); + } +} + +void Task::execute_ready_tasks() +{ + while ( s_ready ) { + + // Remove this task from the ready list + + // Task * task ; + // while ( ! CAS( & s_ready , task = s_ready , s_ready->m_next ) ); + + Task * const task = s_ready ; + s_ready = task->m_next ; + + task->m_next = 0 ; + + // precondition: task->m_state = TASK_STATE_WAITING + // precondition: task->m_dep[i]->m_state == TASK_STATE_COMPLETE for all i + // precondition: does not exist T such that T->m_wait = task + // precondition: does not exist T such that T->m_next = task + + task->m_state = Kokkos::TASK_STATE_EXECUTING ; + + (*task->m_apply)( task ); + + if ( task->m_state == Kokkos::TASK_STATE_EXECUTING ) { + // task did not respawn itself + task->m_state = Kokkos::TASK_STATE_COMPLETE ; + + // release dependences: + for ( int i = 0 ; i < task->m_dep_size ; ++i ) { + assign( task->m_dep + i , 0 ); + } + + // Stop other tasks from adding themselves to 'task->m_wait' ; + + Task * x ; + // CAS( & task->m_wait , x = task->m_wait , s_denied ); + x = task->m_wait ; task->m_wait = s_denied ; + + // update tasks waiting on this task + while ( x ) { + Task * const next = x->m_next ; + + x->m_next = 0 ; + + x->schedule(); // could happen concurrently + + x = next ; + } + } + } +} + +void Task::wait( const Future< void , Kokkos::Serial > & ) +{ execute_ready_tasks(); } + +} // namespace Impl +} // namespace Kokkos + +#endif // defined( KOKKOS_HAVE_SERIAL ) diff --git a/lib/kokkos/core/src/impl/Kokkos_Serial_TaskPolicy.hpp b/lib/kokkos/core/src/impl/Kokkos_Serial_TaskPolicy.hpp new file mode 100755 index 0000000000..bdd9fd03f7 --- /dev/null +++ b/lib/kokkos/core/src/impl/Kokkos_Serial_TaskPolicy.hpp @@ -0,0 +1,763 @@ +/* +//@HEADER +// ************************************************************************ +// +// Kokkos: Manycore Performance-Portable Multidimensional Arrays +// Copyright (2012) Sandia Corporation +// +// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +// the U.S. Government retains certain rights in this software. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// 1. Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// 3. Neither the name of the Corporation nor the names of the +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Questions? Contact H. Carter Edwards (hcedwar@sandia.gov) +// +// ************************************************************************ +//@HEADER +*/ + +// Experimental unified task-data parallel manycore LDRD + +#ifndef KOKKOS_SERIAL_TASKPOLICY_HPP +#define KOKKOS_SERIAL_TASKPOLICY_HPP + +#include +#if defined( KOKKOS_HAVE_SERIAL ) + +#include +#include +#include + +#include +#include +#include + +#include + +//---------------------------------------------------------------------------- +/* Inheritance structure to allow static_cast from the task root type + * and a task's FunctorType. + * + * task_root_type == TaskMember< Space , void , void > + * + * TaskMember< PolicyType , ResultType , FunctorType > + * : TaskMember< PolicyType::Space , ResultType , FunctorType > + * { ... }; + * + * TaskMember< Space , ResultType , FunctorType > + * : TaskMember< Space , ResultType , void > + * , FunctorType + * { ... }; + * + * when ResultType != void + * + * TaskMember< Space , ResultType , void > + * : TaskMember< Space , void , void > + * { ... }; + * + */ +//---------------------------------------------------------------------------- + +namespace Kokkos { +namespace Impl { + +/** \brief Base class for all tasks in the Serial execution space */ +template<> +class TaskMember< Kokkos::Serial , void , void > +{ +public: + + typedef void (* function_apply_type) ( TaskMember * ); + typedef void (* function_dealloc_type)( TaskMember * ); + typedef TaskMember * (* function_verify_type) ( TaskMember * ); + +private: + + const function_dealloc_type m_dealloc ; ///< Deallocation + const function_verify_type m_verify ; ///< Result type verification + const function_apply_type m_apply ; ///< Apply function + TaskMember ** const m_dep ; ///< Dependences + TaskMember * m_wait ; ///< Linked list of tasks waiting on this task + TaskMember * m_next ; ///< Linked list of tasks waiting on a different task + const int m_dep_capacity ; ///< Capacity of dependences + int m_dep_size ; ///< Actual count of dependences + int m_ref_count ; ///< Reference count + int m_state ; ///< State of the task + + // size = 6 Pointers + 4 ints + + TaskMember() /* = delete */ ; + TaskMember( const TaskMember & ) /* = delete */ ; + TaskMember & operator = ( const TaskMember & ) /* = delete */ ; + + static void * allocate( const unsigned arg_sizeof_derived , const unsigned arg_dependence_capacity ); + static void deallocate( void * ); + + void throw_error_add_dependence() const ; + static void throw_error_verify_type(); + + template < class DerivedTaskType > + static + void deallocate( TaskMember * t ) + { + DerivedTaskType * ptr = static_cast< DerivedTaskType * >(t); + ptr->~DerivedTaskType(); + deallocate( (void *) ptr ); + } + +protected : + + ~TaskMember(); + + // Used by TaskMember< Serial , ResultType , void > + TaskMember( const function_verify_type arg_verify + , const function_dealloc_type arg_dealloc + , const function_apply_type arg_apply + , const unsigned arg_sizeof_derived + , const unsigned arg_dependence_capacity + ); + + // Used for TaskMember< Serial , void , void > + TaskMember( const function_dealloc_type arg_dealloc + , const function_apply_type arg_apply + , const unsigned arg_sizeof_derived + , const unsigned arg_dependence_capacity + ); + +public: + + template< typename ResultType > + KOKKOS_FUNCTION static + TaskMember * verify_type( TaskMember * t ) + { + enum { check_type = ! Impl::is_same< ResultType , void >::value }; + + if ( check_type && t != 0 ) { + + // Verify that t->m_verify is this function + const function_verify_type self = & TaskMember::template verify_type< ResultType > ; + + if ( t->m_verify != self ) { + t = 0 ; +#if defined( KOKKOS_ACTIVE_EXECUTION_MEMORY_SPACE_HOST ) + throw_error_verify_type(); +#endif + } + } + return t ; + } + + //---------------------------------------- + /* Inheritence Requirements on task types: + * typedef FunctorType::value_type value_type ; + * class DerivedTaskType + * : public TaskMember< Serial , value_type , FunctorType > + * { ... }; + * class TaskMember< Serial , value_type , FunctorType > + * : public TaskMember< Serial , value_type , void > + * , public Functor + * { ... }; + * If value_type != void + * class TaskMember< Serial , value_type , void > + * : public TaskMember< Serial , void , void > + * + * Allocate space for DerivedTaskType followed by TaskMember*[ dependence_capacity ] + * + */ + + /** \brief Allocate and construct a single-thread task */ + template< class DerivedTaskType > + static + TaskMember * create( const typename DerivedTaskType::functor_type & arg_functor + , const unsigned arg_dependence_capacity ) + { + typedef typename DerivedTaskType::functor_type functor_type ; + typedef typename functor_type::value_type value_type ; + + DerivedTaskType * const task = + new( allocate( sizeof(DerivedTaskType) , arg_dependence_capacity ) ) + DerivedTaskType( & TaskMember::template deallocate< DerivedTaskType > + , & TaskMember::template apply_single< functor_type , value_type > + , sizeof(DerivedTaskType) + , arg_dependence_capacity + , arg_functor ); + + return static_cast< TaskMember * >( task ); + } + + /** \brief Allocate and construct a data parallel task */ + template< class DerivedTaskType > + static + TaskMember * create( const typename DerivedTaskType::policy_type & arg_policy + , const typename DerivedTaskType::functor_type & arg_functor + , const unsigned arg_dependence_capacity ) + { + DerivedTaskType * const task = + new( allocate( sizeof(DerivedTaskType) , arg_dependence_capacity ) ) + DerivedTaskType( & TaskMember::template deallocate< DerivedTaskType > + , sizeof(DerivedTaskType) + , arg_dependence_capacity + , arg_policy + , arg_functor + ); + + return static_cast< TaskMember * >( task ); + } + + void schedule(); + static void execute_ready_tasks(); + static void wait( const Future< void , Kokkos::Serial > & ); + + //---------------------------------------- + + typedef FutureValueTypeIsVoidError get_result_type ; + + KOKKOS_INLINE_FUNCTION + get_result_type get() const { return get_result_type() ; } + + KOKKOS_INLINE_FUNCTION + Kokkos::TaskState get_state() const { return Kokkos::TaskState( m_state ); } + + //---------------------------------------- + +#if defined( KOKKOS_ACTIVE_EXECUTION_MEMORY_SPACE_HOST ) + static + void assign( TaskMember ** const lhs , TaskMember * const rhs , const bool no_throw = false ); +#else + KOKKOS_INLINE_FUNCTION static + void assign( TaskMember ** const lhs , TaskMember * const rhs , const bool no_throw = false ) {} +#endif + + KOKKOS_INLINE_FUNCTION + TaskMember * get_dependence( int i ) const + { return ( Kokkos::TASK_STATE_EXECUTING == m_state && 0 <= i && i < m_dep_size ) ? m_dep[i] : (TaskMember*) 0 ; } + + KOKKOS_INLINE_FUNCTION + int get_dependence() const + { return m_dep_size ; } + + KOKKOS_INLINE_FUNCTION + void clear_dependence() + { + for ( int i = 0 ; i < m_dep_size ; ++i ) assign( m_dep + i , 0 ); + m_dep_size = 0 ; + } + + KOKKOS_INLINE_FUNCTION + void add_dependence( TaskMember * before ) + { + if ( ( Kokkos::TASK_STATE_CONSTRUCTING == m_state || + Kokkos::TASK_STATE_EXECUTING == m_state ) && + m_dep_size < m_dep_capacity ) { + assign( m_dep + m_dep_size , before ); + ++m_dep_size ; + } + else { + throw_error_add_dependence(); + } + } + + //---------------------------------------- + + template< class FunctorType , class ResultType > + KOKKOS_INLINE_FUNCTION static + void apply_single( typename Impl::enable_if< ! Impl::is_same< ResultType , void >::value , TaskMember * >::type t ) + { + typedef TaskMember< Kokkos::Serial , ResultType , FunctorType > derived_type ; + + // TaskMember< Kokkos::Serial , ResultType , FunctorType > + // : public TaskMember< Kokkos::Serial , ResultType , void > + // , public FunctorType + // { ... }; + + derived_type & m = * static_cast< derived_type * >( t ); + + Impl::FunctorApply< FunctorType , void , ResultType & >::apply( (FunctorType &) m , & m.m_result ); + } + + template< class FunctorType , class ResultType > + KOKKOS_INLINE_FUNCTION static + void apply_single( typename Impl::enable_if< Impl::is_same< ResultType , void >::value , TaskMember * >::type t ) + { + typedef TaskMember< Kokkos::Serial , ResultType , FunctorType > derived_type ; + + // TaskMember< Kokkos::Serial , ResultType , FunctorType > + // : public TaskMember< Kokkos::Serial , ResultType , void > + // , public FunctorType + // { ... }; + + derived_type & m = * static_cast< derived_type * >( t ); + + Impl::FunctorApply< FunctorType , void , void >::apply( (FunctorType &) m ); + } +}; + +//---------------------------------------------------------------------------- +/** \brief Base class for tasks with a result value in the Serial execution space. + * + * The FunctorType must be void because this class is accessed by the + * Future class for the task and result value. + * + * Must be derived from TaskMember 'root class' so the Future class + * can correctly static_cast from the 'root class' to this class. + */ +template < class ResultType > +class TaskMember< Kokkos::Serial , ResultType , void > + : public TaskMember< Kokkos::Serial , void , void > +{ +public: + + ResultType m_result ; + + typedef const ResultType & get_result_type ; + + KOKKOS_INLINE_FUNCTION + get_result_type get() const { return m_result ; } + +protected: + + typedef TaskMember< Kokkos::Serial , void , void > task_root_type ; + typedef task_root_type::function_dealloc_type function_dealloc_type ; + typedef task_root_type::function_apply_type function_apply_type ; + + inline + TaskMember( const function_dealloc_type arg_dealloc + , const function_apply_type arg_apply + , const unsigned arg_sizeof_derived + , const unsigned arg_dependence_capacity + ) + : task_root_type( & task_root_type::template verify_type< ResultType > + , arg_dealloc + , arg_apply + , arg_sizeof_derived + , arg_dependence_capacity ) + , m_result() + {} + +}; + +template< class ResultType , class FunctorType > +class TaskMember< Kokkos::Serial , ResultType , FunctorType > + : public TaskMember< Kokkos::Serial , ResultType , void > + , public FunctorType +{ +public: + + typedef FunctorType functor_type ; + + typedef TaskMember< Kokkos::Serial , void , void > task_root_type ; + typedef TaskMember< Kokkos::Serial , ResultType , void > task_base_type ; + typedef task_root_type::function_dealloc_type function_dealloc_type ; + typedef task_root_type::function_apply_type function_apply_type ; + + inline + TaskMember( const function_dealloc_type arg_dealloc + , const function_apply_type arg_apply + , const unsigned arg_sizeof_derived + , const unsigned arg_dependence_capacity + , const functor_type & arg_functor + ) + : task_base_type( arg_dealloc , arg_apply , arg_sizeof_derived , arg_dependence_capacity ) + , functor_type( arg_functor ) + {} +}; + +//---------------------------------------------------------------------------- +/** \brief ForEach task in the Serial execution space + * + * Derived from TaskMember< Kokkos::Serial , ResultType , FunctorType > + * so that Functor can be cast to task root type without knowing policy. + */ +template< class Arg0 , class Arg1 , class Arg2 , class ResultType , class FunctorType > +class TaskForEach< Kokkos::RangePolicy< Arg0 , Arg1 , Arg2 , Kokkos::Serial > + , ResultType + , FunctorType > + : TaskMember< Kokkos::Serial , ResultType , FunctorType > +{ +public: + + typedef FunctorType functor_type ; + typedef RangePolicy< Arg0 , Arg1 , Arg2 , Kokkos::Serial > policy_type ; + +private: + + friend class Kokkos::TaskPolicy< Kokkos::Serial > ; + friend class Kokkos::Impl::TaskMember< Kokkos::Serial , void , void > ; + + typedef TaskMember< Kokkos::Serial , void , void > task_root_type ; + typedef TaskMember< Kokkos::Serial , ResultType , FunctorType > task_base_type ; + typedef task_root_type::function_dealloc_type function_dealloc_type ; + + policy_type m_policy ; + + template< class Tag > + inline + typename Impl::enable_if< Impl::is_same::value >::type + apply_policy() const + { + const typename policy_type::member_type e = m_policy.end(); + for ( typename policy_type::member_type i = m_policy.begin() ; i < e ; ++i ) { + functor_type::operator()(i); + } + } + + template< class Tag > + inline + typename Impl::enable_if< ! Impl::is_same::value >::type + apply_policy() const + { + const Tag tag ; + const typename policy_type::member_type e = m_policy.end(); + for ( typename policy_type::member_type i = m_policy.begin() ; i < e ; ++i ) { + functor_type::operator()(tag,i); + } + } + + static + void apply_parallel( task_root_type * t ) + { + static_cast(t)->template apply_policy< typename policy_type::work_tag >(); + + task_root_type::template apply_single< functor_type , ResultType >( t ); + } + + TaskForEach( const function_dealloc_type arg_dealloc + , const int arg_sizeof_derived + , const int arg_dependence_capacity + , const policy_type & arg_policy + , const functor_type & arg_functor + ) + : task_base_type( arg_dealloc + , & apply_parallel + , arg_sizeof_derived + , arg_dependence_capacity + , arg_functor ) + , m_policy( arg_policy ) + {} + + TaskForEach() /* = delete */ ; + TaskForEach( const TaskForEach & ) /* = delete */ ; + TaskForEach & operator = ( const TaskForEach & ) /* = delete */ ; +}; + +//---------------------------------------------------------------------------- +/** \brief Reduce task in the Serial execution space + * + * Derived from TaskMember< Kokkos::Serial , ResultType , FunctorType > + * so that Functor can be cast to task root type without knowing policy. + */ +template< class Arg0 , class Arg1 , class Arg2 , class ResultType , class FunctorType > +class TaskReduce< Kokkos::RangePolicy< Arg0 , Arg1 , Arg2 , Kokkos::Serial > + , ResultType + , FunctorType > + : TaskMember< Kokkos::Serial , ResultType , FunctorType > +{ +public: + + typedef FunctorType functor_type ; + typedef RangePolicy< Arg0 , Arg1 , Arg2 , Kokkos::Serial > policy_type ; + +private: + + friend class Kokkos::TaskPolicy< Kokkos::Serial > ; + friend class Kokkos::Impl::TaskMember< Kokkos::Serial , void , void > ; + + typedef TaskMember< Kokkos::Serial , void , void > task_root_type ; + typedef TaskMember< Kokkos::Serial , ResultType , FunctorType > task_base_type ; + typedef task_root_type::function_dealloc_type function_dealloc_type ; + + policy_type m_policy ; + + template< class Tag > + inline + void apply_policy( typename Impl::enable_if< Impl::is_same::value , ResultType & >::type result ) const + { + Impl::FunctorValueInit< functor_type , Tag >::init( *this , & result ); + const typename policy_type::member_type e = m_policy.end(); + for ( typename policy_type::member_type i = m_policy.begin() ; i < e ; ++i ) { + functor_type::operator()( i, result ); + } + } + + template< class Tag > + inline + void apply_policy( typename Impl::enable_if< ! Impl::is_same::value , ResultType & >::type result ) const + { + Impl::FunctorValueInit< functor_type , Tag >::init( *this , & result ); + const Tag tag ; + const typename policy_type::member_type e = m_policy.end(); + for ( typename policy_type::member_type i = m_policy.begin() ; i < e ; ++i ) { + functor_type::operator()( tag, i, result ); + } + } + + static + void apply_parallel( task_root_type * t ) + { + TaskReduce * const task = static_cast(t); + + task->template apply_policy< typename policy_type::work_tag >( task->task_base_type::m_result ); + + task_root_type::template apply_single< functor_type , ResultType >( t ); + } + + TaskReduce( const function_dealloc_type arg_dealloc + , const int arg_sizeof_derived + , const int arg_dependence_capacity + , const policy_type & arg_policy + , const functor_type & arg_functor + ) + : task_base_type( arg_dealloc + , & apply_parallel + , arg_sizeof_derived + , arg_dependence_capacity + , arg_functor ) + , m_policy( arg_policy ) + {} + + TaskReduce() /* = delete */ ; + TaskReduce( const TaskReduce & ) /* = delete */ ; + TaskReduce & operator = ( const TaskReduce & ) /* = delete */ ; +}; + +} /* namespace Impl */ +} /* namespace Kokkos */ + +//---------------------------------------------------------------------------- +//---------------------------------------------------------------------------- + +namespace Kokkos { + +template<> +class TaskPolicy< Kokkos::Serial > +{ +public: + + typedef Kokkos::Serial execution_space ; + +private: + + typedef Impl::TaskMember< execution_space , void , void > task_root_type ; + + TaskPolicy & operator = ( const TaskPolicy & ) /* = delete */ ; + + template< class FunctorType > + static inline + const task_root_type * get_task_root( const FunctorType * f ) + { + typedef Impl::TaskMember< execution_space , typename FunctorType::value_type , FunctorType > task_type ; + return static_cast< const task_root_type * >( static_cast< const task_type * >(f) ); + } + + template< class FunctorType > + static inline + task_root_type * get_task_root( FunctorType * f ) + { + typedef Impl::TaskMember< execution_space , typename FunctorType::value_type , FunctorType > task_type ; + return static_cast< task_root_type * >( static_cast< task_type * >(f) ); + } + + const unsigned m_default_dependence_capacity ; + +public: + + KOKKOS_INLINE_FUNCTION + TaskPolicy() : m_default_dependence_capacity(4) {} + + KOKKOS_INLINE_FUNCTION + TaskPolicy( const TaskPolicy & rhs ) : m_default_dependence_capacity( rhs.m_default_dependence_capacity ) {} + + KOKKOS_INLINE_FUNCTION + explicit + TaskPolicy( const unsigned arg_default_dependence_capacity ) + : m_default_dependence_capacity( arg_default_dependence_capacity ) {} + + KOKKOS_INLINE_FUNCTION + TaskPolicy( const TaskPolicy & + , const unsigned arg_default_dependence_capacity ) + : m_default_dependence_capacity( arg_default_dependence_capacity ) {} + + //---------------------------------------- + + template< class ValueType > + KOKKOS_INLINE_FUNCTION + const Future< ValueType , execution_space > & + spawn( const Future< ValueType , execution_space > & f ) const + { +#if defined( KOKKOS_ACTIVE_EXECUTION_MEMORY_SPACE_HOST ) + f.m_task->schedule(); +#endif + return f ; + } + + // Create single-thread task + + template< class FunctorType > + KOKKOS_INLINE_FUNCTION + Future< typename FunctorType::value_type , execution_space > + create( const FunctorType & functor + , const unsigned dependence_capacity = ~0u ) const + { + typedef typename FunctorType::value_type value_type ; + typedef Impl::TaskMember< execution_space , value_type , FunctorType > task_type ; + return Future< value_type , execution_space >( +#if defined( KOKKOS_ACTIVE_EXECUTION_MEMORY_SPACE_HOST ) + task_root_type::create< task_type >( + functor , ( ~0u == dependence_capacity ? m_default_dependence_capacity : dependence_capacity ) ) +#endif + ); + } + + // Create parallel foreach task + + template< class PolicyType , class FunctorType > + KOKKOS_INLINE_FUNCTION + Future< typename FunctorType::value_type , execution_space > + create_foreach( const PolicyType & policy + , const FunctorType & functor + , const unsigned dependence_capacity = ~0u ) const + { + typedef typename FunctorType::value_type value_type ; + typedef Impl::TaskForEach< PolicyType , value_type , FunctorType > task_type ; + return Future< value_type , execution_space >( +#if defined( KOKKOS_ACTIVE_EXECUTION_MEMORY_SPACE_HOST ) + task_root_type::create< task_type >( policy , functor , + ( ~0u == dependence_capacity ? m_default_dependence_capacity : dependence_capacity ) ) +#endif + ); + } + + // Create parallel reduce task + + template< class PolicyType , class FunctorType > + KOKKOS_INLINE_FUNCTION + Future< typename FunctorType::value_type , execution_space > + create_reduce( const PolicyType & policy + , const FunctorType & functor + , const unsigned dependence_capacity = ~0u ) const + { + typedef typename FunctorType::value_type value_type ; + typedef Impl::TaskReduce< PolicyType , value_type , FunctorType > task_type ; + return Future< value_type , execution_space >( +#if defined( KOKKOS_ACTIVE_EXECUTION_MEMORY_SPACE_HOST ) + task_root_type::create< task_type >( policy , functor , + ( ~0u == dependence_capacity ? m_default_dependence_capacity : dependence_capacity ) ) +#endif + ); + } + + // Add dependence + template< class A1 , class A2 , class A3 , class A4 > + KOKKOS_INLINE_FUNCTION + void add_dependence( const Future & after + , const Future & before + , typename Impl::enable_if + < Impl::is_same< typename Future::execution_space , execution_space >::value + && + Impl::is_same< typename Future::execution_space , execution_space >::value + >::type * = 0 + ) const + { +#if defined( KOKKOS_ACTIVE_EXECUTION_MEMORY_SPACE_HOST ) + after.m_task->add_dependence( before.m_task ); +#endif + } + + //---------------------------------------- + // Functions for an executing task functor to query dependences, + // set new dependences, and respawn itself. + + template< class FunctorType > + KOKKOS_INLINE_FUNCTION + Future< void , execution_space > + get_dependence( const FunctorType * task_functor , int i ) const + { + return Future( +#if defined( KOKKOS_ACTIVE_EXECUTION_MEMORY_SPACE_HOST ) + get_task_root(task_functor)->get_dependence(i) +#endif + ); + } + + template< class FunctorType > + KOKKOS_INLINE_FUNCTION + int get_dependence( const FunctorType * task_functor ) const +#if defined( KOKKOS_ACTIVE_EXECUTION_MEMORY_SPACE_HOST ) + { return get_task_root(task_functor)->get_dependence(); } +#else + { return 0 ; } +#endif + + template< class FunctorType > + KOKKOS_INLINE_FUNCTION + void clear_dependence( FunctorType * task_functor ) const +#if defined( KOKKOS_ACTIVE_EXECUTION_MEMORY_SPACE_HOST ) + { get_task_root(task_functor)->clear_dependence(); } +#else + {} +#endif + + template< class FunctorType , class A3 , class A4 > + KOKKOS_INLINE_FUNCTION + void add_dependence( FunctorType * task_functor + , const Future & before + , typename Impl::enable_if + < Impl::is_same< typename Future::execution_space , execution_space >::value + >::type * = 0 + ) const +#if defined( KOKKOS_ACTIVE_EXECUTION_MEMORY_SPACE_HOST ) + { get_task_root(task_functor)->add_dependence( before.m_task ); } +#else + {} +#endif + + template< class FunctorType > + KOKKOS_INLINE_FUNCTION + void respawn( FunctorType * task_functor ) const +#if defined( KOKKOS_ACTIVE_EXECUTION_MEMORY_SPACE_HOST ) + { get_task_root(task_functor)->schedule(); } +#else + {} +#endif +}; + +inline +void wait( TaskPolicy< Kokkos::Serial > & ) +{ Impl::TaskMember< Kokkos::Serial , void , void >::execute_ready_tasks(); } + +inline +void wait( const Future< void , Kokkos::Serial > & future ) +{ Impl::TaskMember< Kokkos::Serial , void , void >::wait( future ); } + +} // namespace Kokkos + +//---------------------------------------------------------------------------- + +#endif /* defined( KOKKOS_HAVE_SERIAL ) */ +#endif /* #define KOKKOS_SERIAL_TASK_HPP */ + diff --git a/lib/kokkos/core/src/impl/Kokkos_Shape.cpp b/lib/kokkos/core/src/impl/Kokkos_Shape.cpp new file mode 100755 index 0000000000..062946b39c --- /dev/null +++ b/lib/kokkos/core/src/impl/Kokkos_Shape.cpp @@ -0,0 +1,178 @@ +/* +//@HEADER +// ************************************************************************ +// +// Kokkos: Manycore Performance-Portable Multidimensional Arrays +// Copyright (2012) Sandia Corporation +// +// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +// the U.S. Government retains certain rights in this software. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// 1. Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// 3. Neither the name of the Corporation nor the names of the +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Questions? Contact H. Carter Edwards (hcedwar@sandia.gov) +// +// ************************************************************************ +//@HEADER +*/ + + +#include +#include +#include + +//---------------------------------------------------------------------------- +//---------------------------------------------------------------------------- + +namespace Kokkos { +namespace Impl { + +void assert_counts_are_equal_throw( + const size_t x_count , + const size_t y_count ) +{ + std::ostringstream msg ; + + msg << "Kokkos::Impl::assert_counts_are_equal_throw( " + << x_count << " != " << y_count << " )" ; + + throw_runtime_exception( msg.str() ); +} + +void assert_shapes_are_equal_throw( + const unsigned x_scalar_size , + const unsigned x_rank , + const size_t x_N0 , const unsigned x_N1 , + const unsigned x_N2 , const unsigned x_N3 , + const unsigned x_N4 , const unsigned x_N5 , + const unsigned x_N6 , const unsigned x_N7 , + + const unsigned y_scalar_size , + const unsigned y_rank , + const size_t y_N0 , const unsigned y_N1 , + const unsigned y_N2 , const unsigned y_N3 , + const unsigned y_N4 , const unsigned y_N5 , + const unsigned y_N6 , const unsigned y_N7 ) +{ + std::ostringstream msg ; + + msg << "Kokkos::Impl::assert_shape_are_equal_throw( {" + << " scalar_size(" << x_scalar_size + << ") rank(" << x_rank + << ") dimension(" ; + if ( 0 < x_rank ) { msg << " " << x_N0 ; } + if ( 1 < x_rank ) { msg << " " << x_N1 ; } + if ( 2 < x_rank ) { msg << " " << x_N2 ; } + if ( 3 < x_rank ) { msg << " " << x_N3 ; } + if ( 4 < x_rank ) { msg << " " << x_N4 ; } + if ( 5 < x_rank ) { msg << " " << x_N5 ; } + if ( 6 < x_rank ) { msg << " " << x_N6 ; } + if ( 7 < x_rank ) { msg << " " << x_N7 ; } + msg << " ) } != { " + << " scalar_size(" << y_scalar_size + << ") rank(" << y_rank + << ") dimension(" ; + if ( 0 < y_rank ) { msg << " " << y_N0 ; } + if ( 1 < y_rank ) { msg << " " << y_N1 ; } + if ( 2 < y_rank ) { msg << " " << y_N2 ; } + if ( 3 < y_rank ) { msg << " " << y_N3 ; } + if ( 4 < y_rank ) { msg << " " << y_N4 ; } + if ( 5 < y_rank ) { msg << " " << y_N5 ; } + if ( 6 < y_rank ) { msg << " " << y_N6 ; } + if ( 7 < y_rank ) { msg << " " << y_N7 ; } + msg << " ) } )" ; + + throw_runtime_exception( msg.str() ); +} + +void AssertShapeBoundsAbort< Kokkos::HostSpace >::apply( + const size_t rank , + const size_t n0 , const size_t n1 , + const size_t n2 , const size_t n3 , + const size_t n4 , const size_t n5 , + const size_t n6 , const size_t n7 , + + const size_t arg_rank , + const size_t i0 , const size_t i1 , + const size_t i2 , const size_t i3 , + const size_t i4 , const size_t i5 , + const size_t i6 , const size_t i7 ) +{ + std::ostringstream msg ; + msg << "Kokkos::Impl::AssertShapeBoundsAbort( shape = {" ; + if ( 0 < rank ) { msg << " " << n0 ; } + if ( 1 < rank ) { msg << " " << n1 ; } + if ( 2 < rank ) { msg << " " << n2 ; } + if ( 3 < rank ) { msg << " " << n3 ; } + if ( 4 < rank ) { msg << " " << n4 ; } + if ( 5 < rank ) { msg << " " << n5 ; } + if ( 6 < rank ) { msg << " " << n6 ; } + if ( 7 < rank ) { msg << " " << n7 ; } + msg << " } index = {" ; + if ( 0 < arg_rank ) { msg << " " << i0 ; } + if ( 1 < arg_rank ) { msg << " " << i1 ; } + if ( 2 < arg_rank ) { msg << " " << i2 ; } + if ( 3 < arg_rank ) { msg << " " << i3 ; } + if ( 4 < arg_rank ) { msg << " " << i4 ; } + if ( 5 < arg_rank ) { msg << " " << i5 ; } + if ( 6 < arg_rank ) { msg << " " << i6 ; } + if ( 7 < arg_rank ) { msg << " " << i7 ; } + msg << " } )" ; + + throw_runtime_exception( msg.str() ); +} + +void assert_shape_effective_rank1_at_leastN_throw( + const size_t x_rank , const size_t x_N0 , + const size_t x_N1 , const size_t x_N2 , + const size_t x_N3 , const size_t x_N4 , + const size_t x_N5 , const size_t x_N6 , + const size_t x_N7 , + const size_t N0 ) +{ + std::ostringstream msg ; + + msg << "Kokkos::Impl::assert_shape_effective_rank1_at_leastN_throw( shape = {" ; + if ( 0 < x_rank ) { msg << " " << x_N0 ; } + if ( 1 < x_rank ) { msg << " " << x_N1 ; } + if ( 2 < x_rank ) { msg << " " << x_N2 ; } + if ( 3 < x_rank ) { msg << " " << x_N3 ; } + if ( 4 < x_rank ) { msg << " " << x_N4 ; } + if ( 5 < x_rank ) { msg << " " << x_N5 ; } + if ( 6 < x_rank ) { msg << " " << x_N6 ; } + if ( 7 < x_rank ) { msg << " " << x_N7 ; } + msg << " } N = " << N0 << " )" ; + + throw_runtime_exception( msg.str() ); +} + + + +} +} + diff --git a/lib/kokkos/core/src/impl/Kokkos_Shape.hpp b/lib/kokkos/core/src/impl/Kokkos_Shape.hpp new file mode 100755 index 0000000000..73be5717ae --- /dev/null +++ b/lib/kokkos/core/src/impl/Kokkos_Shape.hpp @@ -0,0 +1,917 @@ +/* +//@HEADER +// ************************************************************************ +// +// Kokkos: Manycore Performance-Portable Multidimensional Arrays +// Copyright (2012) Sandia Corporation +// +// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +// the U.S. Government retains certain rights in this software. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// 1. Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// 3. Neither the name of the Corporation nor the names of the +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Questions? Contact H. Carter Edwards (hcedwar@sandia.gov) +// +// ************************************************************************ +//@HEADER +*/ + +#ifndef KOKKOS_SHAPE_HPP +#define KOKKOS_SHAPE_HPP + +#include +#include +#include +#include +#include + +//---------------------------------------------------------------------------- +//---------------------------------------------------------------------------- + +namespace Kokkos { +namespace Impl { + +//---------------------------------------------------------------------------- +/** \brief The shape of a Kokkos with dynamic and static dimensions. + * Dynamic dimensions are member values and static dimensions are + * 'static const' values. + * + * The upper bound on the array rank is eight. + */ +template< unsigned ScalarSize , + unsigned Rank , + unsigned s0 = 1 , + unsigned s1 = 1 , + unsigned s2 = 1 , + unsigned s3 = 1 , + unsigned s4 = 1 , + unsigned s5 = 1 , + unsigned s6 = 1 , + unsigned s7 = 1 > +struct Shape ; + +//---------------------------------------------------------------------------- +/** \brief Shape equality if the value type, layout, and dimensions + * are equal. + */ +template< unsigned xSize , unsigned xRank , + unsigned xN0 , unsigned xN1 , unsigned xN2 , unsigned xN3 , + unsigned xN4 , unsigned xN5 , unsigned xN6 , unsigned xN7 , + + unsigned ySize , unsigned yRank , + unsigned yN0 , unsigned yN1 , unsigned yN2 , unsigned yN3 , + unsigned yN4 , unsigned yN5 , unsigned yN6 , unsigned yN7 > +KOKKOS_INLINE_FUNCTION +bool operator == ( const Shape & x , + const Shape & y ) +{ + enum { same_size = xSize == ySize }; + enum { same_rank = xRank == yRank }; + + return same_size && same_rank && + size_t( x.N0 ) == size_t( y.N0 ) && + unsigned( x.N1 ) == unsigned( y.N1 ) && + unsigned( x.N2 ) == unsigned( y.N2 ) && + unsigned( x.N3 ) == unsigned( y.N3 ) && + unsigned( x.N4 ) == unsigned( y.N4 ) && + unsigned( x.N5 ) == unsigned( y.N5 ) && + unsigned( x.N6 ) == unsigned( y.N6 ) && + unsigned( x.N7 ) == unsigned( y.N7 ) ; +} + +template< unsigned xSize , unsigned xRank , + unsigned xN0 , unsigned xN1 , unsigned xN2 , unsigned xN3 , + unsigned xN4 , unsigned xN5 , unsigned xN6 , unsigned xN7 , + + unsigned ySize ,unsigned yRank , + unsigned yN0 , unsigned yN1 , unsigned yN2 , unsigned yN3 , + unsigned yN4 , unsigned yN5 , unsigned yN6 , unsigned yN7 > +KOKKOS_INLINE_FUNCTION +bool operator != ( const Shape & x , + const Shape & y ) +{ return ! operator == ( x , y ); } + +//---------------------------------------------------------------------------- + +void assert_counts_are_equal_throw( + const size_t x_count , + const size_t y_count ); + +inline +void assert_counts_are_equal( + const size_t x_count , + const size_t y_count ) +{ + if ( x_count != y_count ) { + assert_counts_are_equal_throw( x_count , y_count ); + } +} + +void assert_shapes_are_equal_throw( + const unsigned x_scalar_size , + const unsigned x_rank , + const size_t x_N0 , const unsigned x_N1 , + const unsigned x_N2 , const unsigned x_N3 , + const unsigned x_N4 , const unsigned x_N5 , + const unsigned x_N6 , const unsigned x_N7 , + + const unsigned y_scalar_size , + const unsigned y_rank , + const size_t y_N0 , const unsigned y_N1 , + const unsigned y_N2 , const unsigned y_N3 , + const unsigned y_N4 , const unsigned y_N5 , + const unsigned y_N6 , const unsigned y_N7 ); + +template< unsigned xSize , unsigned xRank , + unsigned xN0 , unsigned xN1 , unsigned xN2 , unsigned xN3 , + unsigned xN4 , unsigned xN5 , unsigned xN6 , unsigned xN7 , + + unsigned ySize , unsigned yRank , + unsigned yN0 , unsigned yN1 , unsigned yN2 , unsigned yN3 , + unsigned yN4 , unsigned yN5 , unsigned yN6 , unsigned yN7 > +inline +void assert_shapes_are_equal( + const Shape & x , + const Shape & y ) +{ + typedef Shape x_type ; + typedef Shape y_type ; + + if ( x != y ) { + assert_shapes_are_equal_throw( + x_type::scalar_size, x_type::rank, x.N0, x.N1, x.N2, x.N3, x.N4, x.N5, x.N6, x.N7, + y_type::scalar_size, y_type::rank, y.N0, y.N1, y.N2, y.N3, y.N4, y.N5, y.N6, y.N7 ); + } +} + +template< unsigned xSize , unsigned xRank , + unsigned xN0 , unsigned xN1 , unsigned xN2 , unsigned xN3 , + unsigned xN4 , unsigned xN5 , unsigned xN6 , unsigned xN7 , + + unsigned ySize , unsigned yRank , + unsigned yN0 , unsigned yN1 , unsigned yN2 , unsigned yN3 , + unsigned yN4 , unsigned yN5 , unsigned yN6 , unsigned yN7 > +void assert_shapes_equal_dimension( + const Shape & x , + const Shape & y ) +{ + typedef Shape x_type ; + typedef Shape y_type ; + + // Omit comparison of scalar_size. + if ( unsigned( x.rank ) != unsigned( y.rank ) || + size_t( x.N0 ) != size_t( y.N0 ) || + unsigned( x.N1 ) != unsigned( y.N1 ) || + unsigned( x.N2 ) != unsigned( y.N2 ) || + unsigned( x.N3 ) != unsigned( y.N3 ) || + unsigned( x.N4 ) != unsigned( y.N4 ) || + unsigned( x.N5 ) != unsigned( y.N5 ) || + unsigned( x.N6 ) != unsigned( y.N6 ) || + unsigned( x.N7 ) != unsigned( y.N7 ) ) { + assert_shapes_are_equal_throw( + x_type::scalar_size, x_type::rank, x.N0, x.N1, x.N2, x.N3, x.N4, x.N5, x.N6, x.N7, + y_type::scalar_size, y_type::rank, y.N0, y.N1, y.N2, y.N3, y.N4, y.N5, y.N6, y.N7 ); + } +} + +//---------------------------------------------------------------------------- + +template< class ShapeType > struct assert_shape_is_rank_zero ; +template< class ShapeType > struct assert_shape_is_rank_one ; + +template< unsigned Size > +struct assert_shape_is_rank_zero< Shape > + : public true_type {}; + +template< unsigned Size , unsigned s0 > +struct assert_shape_is_rank_one< Shape > + : public true_type {}; + +//---------------------------------------------------------------------------- + +/** \brief Array bounds assertion templated on the execution space + * to allow device-specific abort code. + */ +template< class Space > +struct AssertShapeBoundsAbort ; + +template<> +struct AssertShapeBoundsAbort< Kokkos::HostSpace > +{ + static void apply( const size_t rank , + const size_t n0 , const size_t n1 , + const size_t n2 , const size_t n3 , + const size_t n4 , const size_t n5 , + const size_t n6 , const size_t n7 , + const size_t arg_rank , + const size_t i0 , const size_t i1 , + const size_t i2 , const size_t i3 , + const size_t i4 , const size_t i5 , + const size_t i6 , const size_t i7 ); +}; + +template< class ExecutionSpace > +struct AssertShapeBoundsAbort +{ + KOKKOS_INLINE_FUNCTION + static void apply( const size_t rank , + const size_t n0 , const size_t n1 , + const size_t n2 , const size_t n3 , + const size_t n4 , const size_t n5 , + const size_t n6 , const size_t n7 , + const size_t arg_rank , + const size_t i0 , const size_t i1 , + const size_t i2 , const size_t i3 , + const size_t i4 , const size_t i5 , + const size_t i6 , const size_t i7 ) + { + AssertShapeBoundsAbort< Kokkos::HostSpace > + ::apply( rank , n0 , n1 , n2 , n3 , n4 , n5 , n6 , n7 , + arg_rank, i0 , i1 , i2 , i3 , i4 , i5 , i6 , i7 ); + } +}; + +template< class ShapeType > +KOKKOS_INLINE_FUNCTION +void assert_shape_bounds( const ShapeType & shape , + const size_t arg_rank , + const size_t i0 , + const size_t i1 = 0 , + const size_t i2 = 0 , + const size_t i3 = 0 , + const size_t i4 = 0 , + const size_t i5 = 0 , + const size_t i6 = 0 , + const size_t i7 = 0 ) +{ + // Must supply at least as many indices as ranks. + // Every index must be within bounds. + const bool ok = ShapeType::rank <= arg_rank && + i0 < shape.N0 && + i1 < shape.N1 && + i2 < shape.N2 && + i3 < shape.N3 && + i4 < shape.N4 && + i5 < shape.N5 && + i6 < shape.N6 && + i7 < shape.N7 ; + + if ( ! ok ) { + AssertShapeBoundsAbort< Kokkos::Impl::ActiveExecutionMemorySpace > + ::apply( ShapeType::rank , + shape.N0 , shape.N1 , shape.N2 , shape.N3 , + shape.N4 , shape.N5 , shape.N6 , shape.N7 , + arg_rank , i0 , i1 , i2 , i3 , i4 , i5 , i6 , i7 ); + } +} + +#if defined( KOKKOS_EXPRESSION_CHECK ) +#define KOKKOS_ASSERT_SHAPE_BOUNDS_1( S , I0 ) assert_shape_bounds(S,1,I0); +#define KOKKOS_ASSERT_SHAPE_BOUNDS_2( S , I0 , I1 ) assert_shape_bounds(S,2,I0,I1); +#define KOKKOS_ASSERT_SHAPE_BOUNDS_3( S , I0 , I1 , I2 ) assert_shape_bounds(S,3,I0,I1,I2); +#define KOKKOS_ASSERT_SHAPE_BOUNDS_4( S , I0 , I1 , I2 , I3 ) assert_shape_bounds(S,4,I0,I1,I2,I3); +#define KOKKOS_ASSERT_SHAPE_BOUNDS_5( S , I0 , I1 , I2 , I3 , I4 ) assert_shape_bounds(S,5,I0,I1,I2,I3,I4); +#define KOKKOS_ASSERT_SHAPE_BOUNDS_6( S , I0 , I1 , I2 , I3 , I4 , I5 ) assert_shape_bounds(S,6,I0,I1,I2,I3,I4,I5); +#define KOKKOS_ASSERT_SHAPE_BOUNDS_7( S , I0 , I1 , I2 , I3 , I4 , I5 , I6 ) assert_shape_bounds(S,7,I0,I1,I2,I3,I4,I5,I6); +#define KOKKOS_ASSERT_SHAPE_BOUNDS_8( S , I0 , I1 , I2 , I3 , I4 , I5 , I6 , I7 ) assert_shape_bounds(S,8,I0,I1,I2,I3,I4,I5,I6,I7); +#else +#define KOKKOS_ASSERT_SHAPE_BOUNDS_1( S , I0 ) /* */ +#define KOKKOS_ASSERT_SHAPE_BOUNDS_2( S , I0 , I1 ) /* */ +#define KOKKOS_ASSERT_SHAPE_BOUNDS_3( S , I0 , I1 , I2 ) /* */ +#define KOKKOS_ASSERT_SHAPE_BOUNDS_4( S , I0 , I1 , I2 , I3 ) /* */ +#define KOKKOS_ASSERT_SHAPE_BOUNDS_5( S , I0 , I1 , I2 , I3 , I4 ) /* */ +#define KOKKOS_ASSERT_SHAPE_BOUNDS_6( S , I0 , I1 , I2 , I3 , I4 , I5 ) /* */ +#define KOKKOS_ASSERT_SHAPE_BOUNDS_7( S , I0 , I1 , I2 , I3 , I4 , I5 , I6 ) /* */ +#define KOKKOS_ASSERT_SHAPE_BOUNDS_8( S , I0 , I1 , I2 , I3 , I4 , I5 , I6 , I7 ) /* */ +#endif + + +//---------------------------------------------------------------------------- +//---------------------------------------------------------------------------- +// Specialization and optimization for the Rank 0 shape. + +template < unsigned ScalarSize > +struct Shape< ScalarSize , 0, 1,1,1,1, 1,1,1,1 > +{ + enum { scalar_size = ScalarSize }; + enum { rank_dynamic = 0 }; + enum { rank = 0 }; + + enum { N0 = 1 }; + enum { N1 = 1 }; + enum { N2 = 1 }; + enum { N3 = 1 }; + enum { N4 = 1 }; + enum { N5 = 1 }; + enum { N6 = 1 }; + enum { N7 = 1 }; + + KOKKOS_INLINE_FUNCTION + static + void assign( Shape & , + unsigned = 0 , unsigned = 0 , unsigned = 0 , unsigned = 0 , + unsigned = 0 , unsigned = 0 , unsigned = 0 , unsigned = 0 ) + {} +}; + +//---------------------------------------------------------------------------- + +template< unsigned R > struct assign_shape_dimension ; + +#define KOKKOS_ASSIGN_SHAPE_DIMENSION( R ) \ +template<> \ +struct assign_shape_dimension< R > \ +{ \ + template< class ShapeType > \ + KOKKOS_INLINE_FUNCTION \ + assign_shape_dimension( ShapeType & shape \ + , typename Impl::enable_if<( R < ShapeType::rank_dynamic ), size_t >::type n \ + ) { shape.N ## R = n ; } \ +}; + +KOKKOS_ASSIGN_SHAPE_DIMENSION(0) +KOKKOS_ASSIGN_SHAPE_DIMENSION(1) +KOKKOS_ASSIGN_SHAPE_DIMENSION(2) +KOKKOS_ASSIGN_SHAPE_DIMENSION(3) +KOKKOS_ASSIGN_SHAPE_DIMENSION(4) +KOKKOS_ASSIGN_SHAPE_DIMENSION(5) +KOKKOS_ASSIGN_SHAPE_DIMENSION(6) +KOKKOS_ASSIGN_SHAPE_DIMENSION(7) + +#undef KOKKOS_ASSIGN_SHAPE_DIMENSION + +//---------------------------------------------------------------------------- +// All-static dimension array + +template < unsigned ScalarSize , + unsigned Rank , + unsigned s0 , + unsigned s1 , + unsigned s2 , + unsigned s3 , + unsigned s4 , + unsigned s5 , + unsigned s6 , + unsigned s7 > +struct Shape { + + enum { scalar_size = ScalarSize }; + enum { rank_dynamic = 0 }; + enum { rank = Rank }; + + enum { N0 = s0 }; + enum { N1 = s1 }; + enum { N2 = s2 }; + enum { N3 = s3 }; + enum { N4 = s4 }; + enum { N5 = s5 }; + enum { N6 = s6 }; + enum { N7 = s7 }; + + KOKKOS_INLINE_FUNCTION + static + void assign( Shape & , + unsigned = 0 , unsigned = 0 , unsigned = 0 , unsigned = 0 , + unsigned = 0 , unsigned = 0 , unsigned = 0 , unsigned = 0 ) + {} +}; + +// 1 == dynamic_rank <= rank <= 8 +template < unsigned ScalarSize , + unsigned Rank , + unsigned s1 , + unsigned s2 , + unsigned s3 , + unsigned s4 , + unsigned s5 , + unsigned s6 , + unsigned s7 > +struct Shape< ScalarSize , Rank , 0,s1,s2,s3, s4,s5,s6,s7 > +{ + enum { scalar_size = ScalarSize }; + enum { rank_dynamic = 1 }; + enum { rank = Rank }; + + size_t N0 ; // For 1 == dynamic_rank allow N0 > 2^32 + + enum { N1 = s1 }; + enum { N2 = s2 }; + enum { N3 = s3 }; + enum { N4 = s4 }; + enum { N5 = s5 }; + enum { N6 = s6 }; + enum { N7 = s7 }; + + KOKKOS_INLINE_FUNCTION + static + void assign( Shape & s , + size_t n0 , unsigned = 0 , unsigned = 0 , unsigned = 0 , + unsigned = 0 , unsigned = 0 , unsigned = 0 , unsigned = 0 ) + { s.N0 = n0 ; } +}; + +// 2 == dynamic_rank <= rank <= 8 +template < unsigned ScalarSize , unsigned Rank , + unsigned s2 , + unsigned s3 , + unsigned s4 , + unsigned s5 , + unsigned s6 , + unsigned s7 > +struct Shape< ScalarSize , Rank , 0,0,s2,s3, s4,s5,s6,s7 > +{ + enum { scalar_size = ScalarSize }; + enum { rank_dynamic = 2 }; + enum { rank = Rank }; + + unsigned N0 ; + unsigned N1 ; + + enum { N2 = s2 }; + enum { N3 = s3 }; + enum { N4 = s4 }; + enum { N5 = s5 }; + enum { N6 = s6 }; + enum { N7 = s7 }; + + KOKKOS_INLINE_FUNCTION + static + void assign( Shape & s , + unsigned n0 , unsigned n1 , unsigned = 0 , unsigned = 0 , + unsigned = 0 , unsigned = 0 , unsigned = 0 , unsigned = 0 ) + { s.N0 = n0 ; s.N1 = n1 ; } +}; + +// 3 == dynamic_rank <= rank <= 8 +template < unsigned Rank , unsigned ScalarSize , + unsigned s3 , + unsigned s4 , + unsigned s5 , + unsigned s6 , + unsigned s7 > +struct Shape< ScalarSize , Rank , 0,0,0,s3, s4,s5,s6,s7> +{ + enum { scalar_size = ScalarSize }; + enum { rank_dynamic = 3 }; + enum { rank = Rank }; + + unsigned N0 ; + unsigned N1 ; + unsigned N2 ; + + enum { N3 = s3 }; + enum { N4 = s4 }; + enum { N5 = s5 }; + enum { N6 = s6 }; + enum { N7 = s7 }; + + KOKKOS_INLINE_FUNCTION + static + void assign( Shape & s , + unsigned n0 , unsigned n1 , unsigned n2 , unsigned = 0 , + unsigned = 0 , unsigned = 0 , unsigned = 0 , unsigned = 0 ) + { s.N0 = n0 ; s.N1 = n1 ; s.N2 = n2 ; } +}; + +// 4 == dynamic_rank <= rank <= 8 +template < unsigned ScalarSize , unsigned Rank , + unsigned s4 , + unsigned s5 , + unsigned s6 , + unsigned s7 > +struct Shape< ScalarSize , Rank, 0,0,0,0, s4,s5,s6,s7 > +{ + enum { scalar_size = ScalarSize }; + enum { rank_dynamic = 4 }; + enum { rank = Rank }; + + unsigned N0 ; + unsigned N1 ; + unsigned N2 ; + unsigned N3 ; + + enum { N4 = s4 }; + enum { N5 = s5 }; + enum { N6 = s6 }; + enum { N7 = s7 }; + + KOKKOS_INLINE_FUNCTION + static + void assign( Shape & s , + unsigned n0 , unsigned n1 , unsigned n2 , unsigned n3 , + unsigned = 0 , unsigned = 0 , unsigned = 0 , unsigned = 0 ) + { s.N0 = n0 ; s.N1 = n1 ; s.N2 = n2 ; s.N3 = n3 ; } +}; + +// 5 == dynamic_rank <= rank <= 8 +template < unsigned ScalarSize , unsigned Rank , + unsigned s5 , + unsigned s6 , + unsigned s7 > +struct Shape< ScalarSize , Rank , 0,0,0,0, 0,s5,s6,s7 > +{ + enum { scalar_size = ScalarSize }; + enum { rank_dynamic = 5 }; + enum { rank = Rank }; + + unsigned N0 ; + unsigned N1 ; + unsigned N2 ; + unsigned N3 ; + unsigned N4 ; + + enum { N5 = s5 }; + enum { N6 = s6 }; + enum { N7 = s7 }; + + KOKKOS_INLINE_FUNCTION + static + void assign( Shape & s , + unsigned n0 , unsigned n1 , unsigned n2 , unsigned n3 , + unsigned n4 , unsigned = 0 , unsigned = 0 , unsigned = 0 ) + { s.N0 = n0 ; s.N1 = n1 ; s.N2 = n2 ; s.N3 = n3 ; s.N4 = n4 ; } +}; + +// 6 == dynamic_rank <= rank <= 8 +template < unsigned ScalarSize , unsigned Rank , + unsigned s6 , + unsigned s7 > +struct Shape< ScalarSize , Rank , 0,0,0,0, 0,0,s6,s7 > +{ + enum { scalar_size = ScalarSize }; + enum { rank_dynamic = 6 }; + enum { rank = Rank }; + + unsigned N0 ; + unsigned N1 ; + unsigned N2 ; + unsigned N3 ; + unsigned N4 ; + unsigned N5 ; + + enum { N6 = s6 }; + enum { N7 = s7 }; + + KOKKOS_INLINE_FUNCTION + static + void assign( Shape & s , + unsigned n0 , unsigned n1 , unsigned n2 , unsigned n3 , + unsigned n4 , unsigned n5 = 0 , unsigned = 0 , unsigned = 0 ) + { + s.N0 = n0 ; s.N1 = n1 ; s.N2 = n2 ; s.N3 = n3 ; + s.N4 = n4 ; s.N5 = n5 ; + } +}; + +// 7 == dynamic_rank <= rank <= 8 +template < unsigned ScalarSize , unsigned Rank , + unsigned s7 > +struct Shape< ScalarSize , Rank , 0,0,0,0, 0,0,0,s7 > +{ + enum { scalar_size = ScalarSize }; + enum { rank_dynamic = 7 }; + enum { rank = Rank }; + + unsigned N0 ; + unsigned N1 ; + unsigned N2 ; + unsigned N3 ; + unsigned N4 ; + unsigned N5 ; + unsigned N6 ; + + enum { N7 = s7 }; + + KOKKOS_INLINE_FUNCTION + static + void assign( Shape & s , + unsigned n0 , unsigned n1 , unsigned n2 , unsigned n3 , + unsigned n4 , unsigned n5 , unsigned n6 , unsigned = 0 ) + { + s.N0 = n0 ; s.N1 = n1 ; s.N2 = n2 ; s.N3 = n3 ; + s.N4 = n4 ; s.N5 = n5 ; s.N6 = n6 ; + } +}; + +// 8 == dynamic_rank <= rank <= 8 +template < unsigned ScalarSize > +struct Shape< ScalarSize , 8 , 0,0,0,0, 0,0,0,0 > +{ + enum { scalar_size = ScalarSize }; + enum { rank_dynamic = 8 }; + enum { rank = 8 }; + + unsigned N0 ; + unsigned N1 ; + unsigned N2 ; + unsigned N3 ; + unsigned N4 ; + unsigned N5 ; + unsigned N6 ; + unsigned N7 ; + + KOKKOS_INLINE_FUNCTION + static + void assign( Shape & s , + unsigned n0 , unsigned n1 , unsigned n2 , unsigned n3 , + unsigned n4 , unsigned n5 , unsigned n6 , unsigned n7 ) + { + s.N0 = n0 ; s.N1 = n1 ; s.N2 = n2 ; s.N3 = n3 ; + s.N4 = n4 ; s.N5 = n5 ; s.N6 = n6 ; s.N7 = n7 ; + } +}; + +//---------------------------------------------------------------------------- + +template< class ShapeType , unsigned N , + unsigned R = ShapeType::rank_dynamic > +struct ShapeInsert ; + +template< class ShapeType , unsigned N > +struct ShapeInsert< ShapeType , N , 0 > +{ + typedef Shape< ShapeType::scalar_size , + ShapeType::rank + 1 , + N , + ShapeType::N0 , + ShapeType::N1 , + ShapeType::N2 , + ShapeType::N3 , + ShapeType::N4 , + ShapeType::N5 , + ShapeType::N6 > type ; +}; + +template< class ShapeType , unsigned N > +struct ShapeInsert< ShapeType , N , 1 > +{ + typedef Shape< ShapeType::scalar_size , + ShapeType::rank + 1 , + 0 , + N , + ShapeType::N1 , + ShapeType::N2 , + ShapeType::N3 , + ShapeType::N4 , + ShapeType::N5 , + ShapeType::N6 > type ; +}; + +template< class ShapeType , unsigned N > +struct ShapeInsert< ShapeType , N , 2 > +{ + typedef Shape< ShapeType::scalar_size , + ShapeType::rank + 1 , + 0 , + 0 , + N , + ShapeType::N2 , + ShapeType::N3 , + ShapeType::N4 , + ShapeType::N5 , + ShapeType::N6 > type ; +}; + +template< class ShapeType , unsigned N > +struct ShapeInsert< ShapeType , N , 3 > +{ + typedef Shape< ShapeType::scalar_size , + ShapeType::rank + 1 , + 0 , + 0 , + 0 , + N , + ShapeType::N3 , + ShapeType::N4 , + ShapeType::N5 , + ShapeType::N6 > type ; +}; + +template< class ShapeType , unsigned N > +struct ShapeInsert< ShapeType , N , 4 > +{ + typedef Shape< ShapeType::scalar_size , + ShapeType::rank + 1 , + 0 , + 0 , + 0 , + 0 , + N , + ShapeType::N4 , + ShapeType::N5 , + ShapeType::N6 > type ; +}; + +template< class ShapeType , unsigned N > +struct ShapeInsert< ShapeType , N , 5 > +{ + typedef Shape< ShapeType::scalar_size , + ShapeType::rank + 1 , + 0 , + 0 , + 0 , + 0 , + 0 , + N , + ShapeType::N5 , + ShapeType::N6 > type ; +}; + +template< class ShapeType , unsigned N > +struct ShapeInsert< ShapeType , N , 6 > +{ + typedef Shape< ShapeType::scalar_size , + ShapeType::rank + 1 , + 0 , + 0 , + 0 , + 0 , + 0 , + 0 , + N , + ShapeType::N6 > type ; +}; + +template< class ShapeType , unsigned N > +struct ShapeInsert< ShapeType , N , 7 > +{ + typedef Shape< ShapeType::scalar_size , + ShapeType::rank + 1 , + 0 , + 0 , + 0 , + 0 , + 0 , + 0 , + 0 , + N > type ; +}; + +//---------------------------------------------------------------------------- + +template< class DstShape , class SrcShape , + unsigned DstRankDynamic = DstShape::rank_dynamic , + bool DstRankDynamicOK = unsigned(DstShape::rank_dynamic) >= unsigned(SrcShape::rank_dynamic) > +struct ShapeCompatible { enum { value = false }; }; + +template< class DstShape , class SrcShape > +struct ShapeCompatible< DstShape , SrcShape , 8 , true > +{ + enum { value = unsigned(DstShape::scalar_size) == unsigned(SrcShape::scalar_size) }; +}; + +template< class DstShape , class SrcShape > +struct ShapeCompatible< DstShape , SrcShape , 7 , true > +{ + enum { value = unsigned(DstShape::scalar_size) == unsigned(SrcShape::scalar_size) && + unsigned(DstShape::N7) == unsigned(SrcShape::N7) }; +}; + +template< class DstShape , class SrcShape > +struct ShapeCompatible< DstShape , SrcShape , 6 , true > +{ + enum { value = unsigned(DstShape::scalar_size) == unsigned(SrcShape::scalar_size) && + unsigned(DstShape::N6) == unsigned(SrcShape::N6) && + unsigned(DstShape::N7) == unsigned(SrcShape::N7) }; +}; + +template< class DstShape , class SrcShape > +struct ShapeCompatible< DstShape , SrcShape , 5 , true > +{ + enum { value = unsigned(DstShape::scalar_size) == unsigned(SrcShape::scalar_size) && + unsigned(DstShape::N5) == unsigned(SrcShape::N5) && + unsigned(DstShape::N6) == unsigned(SrcShape::N6) && + unsigned(DstShape::N7) == unsigned(SrcShape::N7) }; +}; + +template< class DstShape , class SrcShape > +struct ShapeCompatible< DstShape , SrcShape , 4 , true > +{ + enum { value = unsigned(DstShape::scalar_size) == unsigned(SrcShape::scalar_size) && + unsigned(DstShape::N4) == unsigned(SrcShape::N4) && + unsigned(DstShape::N5) == unsigned(SrcShape::N5) && + unsigned(DstShape::N6) == unsigned(SrcShape::N6) && + unsigned(DstShape::N7) == unsigned(SrcShape::N7) }; +}; + +template< class DstShape , class SrcShape > +struct ShapeCompatible< DstShape , SrcShape , 3 , true > +{ + enum { value = unsigned(DstShape::scalar_size) == unsigned(SrcShape::scalar_size) && + unsigned(DstShape::N3) == unsigned(SrcShape::N3) && + unsigned(DstShape::N4) == unsigned(SrcShape::N4) && + unsigned(DstShape::N5) == unsigned(SrcShape::N5) && + unsigned(DstShape::N6) == unsigned(SrcShape::N6) && + unsigned(DstShape::N7) == unsigned(SrcShape::N7) }; +}; + +template< class DstShape , class SrcShape > +struct ShapeCompatible< DstShape , SrcShape , 2 , true > +{ + enum { value = unsigned(DstShape::scalar_size) == unsigned(SrcShape::scalar_size) && + unsigned(DstShape::N2) == unsigned(SrcShape::N2) && + unsigned(DstShape::N3) == unsigned(SrcShape::N3) && + unsigned(DstShape::N4) == unsigned(SrcShape::N4) && + unsigned(DstShape::N5) == unsigned(SrcShape::N5) && + unsigned(DstShape::N6) == unsigned(SrcShape::N6) && + unsigned(DstShape::N7) == unsigned(SrcShape::N7) }; +}; + +template< class DstShape , class SrcShape > +struct ShapeCompatible< DstShape , SrcShape , 1 , true > +{ + enum { value = unsigned(DstShape::scalar_size) == unsigned(SrcShape::scalar_size) && + unsigned(DstShape::N1) == unsigned(SrcShape::N1) && + unsigned(DstShape::N2) == unsigned(SrcShape::N2) && + unsigned(DstShape::N3) == unsigned(SrcShape::N3) && + unsigned(DstShape::N4) == unsigned(SrcShape::N4) && + unsigned(DstShape::N5) == unsigned(SrcShape::N5) && + unsigned(DstShape::N6) == unsigned(SrcShape::N6) && + unsigned(DstShape::N7) == unsigned(SrcShape::N7) }; +}; + +template< class DstShape , class SrcShape > +struct ShapeCompatible< DstShape , SrcShape , 0 , true > +{ + enum { value = unsigned(DstShape::scalar_size) == unsigned(SrcShape::scalar_size) && + unsigned(DstShape::N0) == unsigned(SrcShape::N0) && + unsigned(DstShape::N1) == unsigned(SrcShape::N1) && + unsigned(DstShape::N2) == unsigned(SrcShape::N2) && + unsigned(DstShape::N3) == unsigned(SrcShape::N3) && + unsigned(DstShape::N4) == unsigned(SrcShape::N4) && + unsigned(DstShape::N5) == unsigned(SrcShape::N5) && + unsigned(DstShape::N6) == unsigned(SrcShape::N6) && + unsigned(DstShape::N7) == unsigned(SrcShape::N7) }; +}; + +} /* namespace Impl */ +} /* namespace Kokkos */ + +//---------------------------------------------------------------------------- +//---------------------------------------------------------------------------- + +namespace Kokkos { +namespace Impl { + +template< unsigned ScalarSize , unsigned Rank , + unsigned s0 , unsigned s1 , unsigned s2 , unsigned s3 , + unsigned s4 , unsigned s5 , unsigned s6 , unsigned s7 , + typename iType > +KOKKOS_INLINE_FUNCTION +size_t dimension( + const Shape & shape , + const iType & r ) +{ + return 0 == r ? shape.N0 : ( + 1 == r ? shape.N1 : ( + 2 == r ? shape.N2 : ( + 3 == r ? shape.N3 : ( + 4 == r ? shape.N4 : ( + 5 == r ? shape.N5 : ( + 6 == r ? shape.N6 : ( + 7 == r ? shape.N7 : 1 ))))))); +} + +template< unsigned ScalarSize , unsigned Rank , + unsigned s0 , unsigned s1 , unsigned s2 , unsigned s3 , + unsigned s4 , unsigned s5 , unsigned s6 , unsigned s7 > +KOKKOS_INLINE_FUNCTION +size_t cardinality_count( + const Shape & shape ) +{ + return size_t(shape.N0) * shape.N1 * shape.N2 * shape.N3 * + shape.N4 * shape.N5 * shape.N6 * shape.N7 ; +} + +//---------------------------------------------------------------------------- + +} /* namespace Impl */ +} /* namespace Kokkos */ + +#endif /* #ifndef KOKKOS_CORESHAPE_HPP */ + diff --git a/lib/kokkos/core/src/impl/Kokkos_StaticAssert.hpp b/lib/kokkos/core/src/impl/Kokkos_StaticAssert.hpp new file mode 100755 index 0000000000..f1017c312f --- /dev/null +++ b/lib/kokkos/core/src/impl/Kokkos_StaticAssert.hpp @@ -0,0 +1,79 @@ +/* +//@HEADER +// ************************************************************************ +// +// Kokkos: Manycore Performance-Portable Multidimensional Arrays +// Copyright (2012) Sandia Corporation +// +// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +// the U.S. Government retains certain rights in this software. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// 1. Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// 3. Neither the name of the Corporation nor the names of the +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Questions? Contact H. Carter Edwards (hcedwar@sandia.gov) +// +// ************************************************************************ +//@HEADER +*/ + +#ifndef KOKKOS_STATICASSERT_HPP +#define KOKKOS_STATICASSERT_HPP + +namespace Kokkos { +namespace Impl { + +template < bool , class T = void > +struct StaticAssert ; + +template< class T > +struct StaticAssert< true , T > { + typedef T type ; + static const bool value = true ; +}; + +template < class A , class B > +struct StaticAssertSame ; + +template < class A > +struct StaticAssertSame { typedef A type ; }; + +template < class A , class B > +struct StaticAssertAssignable ; + +template < class A > +struct StaticAssertAssignable { typedef A type ; }; + +template < class A > +struct StaticAssertAssignable< const A , A > { typedef const A type ; }; + +} // namespace Impl +} // namespace Kokkos + +#endif /* KOKKOS_STATICASSERT_HPP */ + + diff --git a/lib/kokkos/core/src/impl/Kokkos_Tags.hpp b/lib/kokkos/core/src/impl/Kokkos_Tags.hpp new file mode 100755 index 0000000000..372ea14b6e --- /dev/null +++ b/lib/kokkos/core/src/impl/Kokkos_Tags.hpp @@ -0,0 +1,131 @@ +/* +//@HEADER +// ************************************************************************ +// +// Kokkos +// Manycore Performance-Portable Multidimensional Arrays +// +// Copyright (2012) Sandia Corporation +// +// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +// the U.S. Government retains certain rights in this software. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// 1. Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// 3. Neither the name of the Corporation nor the names of the +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Questions? Contact H. Carter Edwards (hcedwar@sandia.gov) +// +// ************************************************************************ +//@HEADER +*/ + +#ifndef KOKKOS_TAGS_HPP +#define KOKKOS_TAGS_HPP + +#include + +//---------------------------------------------------------------------------- +//---------------------------------------------------------------------------- + +namespace Kokkos { +namespace Impl { + +struct LayoutTag {}; + +struct MemorySpaceTag {}; +struct MemoryTraitsTag {}; + +struct ExecutionPolicyTag {}; +struct ExecutionSpaceTag {}; + + +template< class C , class Enable = void > +struct is_memory_space : public bool_< false > {}; + +template< class C , class Enable = void > +struct is_execution_space : public bool_< false > {}; + +template< class C , class Enable = void > +struct is_execution_policy : public bool_< false > {}; + +template< class C , class Enable = void > +struct is_array_layout : public Impl::false_type {}; + +template< class C , class Enable = void > +struct is_memory_traits : public Impl::false_type {}; + + +template< class C > +struct is_memory_space< C , typename Impl::enable_if_type< typename C::memory_space >::type > + : public bool_< Impl::is_same< C , typename C::memory_space >::value > {}; + +template< class C > +struct is_execution_space< C , typename Impl::enable_if_type< typename C::execution_space >::type > + : public bool_< Impl::is_same< C , typename C::execution_space >::value > {}; + +template< class C > +struct is_execution_policy< C , typename Impl::enable_if_type< typename C::execution_policy >::type > + : public bool_< Impl::is_same< C , typename C::execution_policy >::value > {}; + +template< class C > +struct is_array_layout< C , typename Impl::enable_if_type< typename C::array_layout >::type > + : public bool_< Impl::is_same< C , typename C::array_layout >::value > {}; + +template< class C > +struct is_memory_traits< C , typename Impl::enable_if_type< typename C::memory_traits >::type > + : public bool_< Impl::is_same< C , typename C::memory_traits >::value > {}; + +//---------------------------------------------------------------------------- + +template< class C , class Enable = void > +struct is_space : public Impl::false_type {}; + +template< class C > +struct is_space< C + , typename Impl::enable_if<( + Impl::is_same< C , typename C::execution_space >::value || + Impl::is_same< C , typename C::memory_space >::value + )>::type + > + : public Impl::true_type +{ + typedef typename C::execution_space execution_space ; + typedef typename C::memory_space memory_space ; + + // The host_mirror_space defines a space with host-resident memory. + // If the execution space's memory space is HostSpace then use that execution space. + // Else use the HostSpace. + typedef + typename Impl::if_c< Impl::is_same< typename execution_space::memory_space , HostSpace >::value , execution_space , + HostSpace >::type + host_mirror_space ; +}; + +} +} + +#endif diff --git a/lib/kokkos/core/src/impl/Kokkos_Timer.hpp b/lib/kokkos/core/src/impl/Kokkos_Timer.hpp new file mode 100755 index 0000000000..17a5b2c9bb --- /dev/null +++ b/lib/kokkos/core/src/impl/Kokkos_Timer.hpp @@ -0,0 +1,115 @@ +/* +//@HEADER +// ************************************************************************ +// +// Kokkos: Manycore Performance-Portable Multidimensional Arrays +// Copyright (2012) Sandia Corporation +// +// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +// the U.S. Government retains certain rights in this software. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// 1. Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// 3. Neither the name of the Corporation nor the names of the +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Questions? Contact H. Carter Edwards (hcedwar@sandia.gov) +// +// ************************************************************************ +//@HEADER +*/ + +#ifndef KOKKOS_IMPLWALLTIME_HPP +#define KOKKOS_IMPLWALLTIME_HPP + +#include + +#ifdef _MSC_VER +#undef KOKKOS_USE_LIBRT +#include +#else +#ifdef KOKKOS_USE_LIBRT +#include +#else +#include +#endif +#endif + +namespace Kokkos { +namespace Impl { + +/** \brief Time since construction */ + +class Timer { +private: + #ifdef KOKKOS_USE_LIBRT + struct timespec m_old; + #else + struct timeval m_old ; + #endif + Timer( const Timer & ); + Timer & operator = ( const Timer & ); +public: + + inline + void reset() { + #ifdef KOKKOS_USE_LIBRT + clock_gettime(CLOCK_REALTIME, &m_old); + #else + gettimeofday( & m_old , ((struct timezone *) NULL ) ); + #endif + } + + inline + ~Timer() {} + + inline + Timer() { reset(); } + + inline + double seconds() const + { + #ifdef KOKKOS_USE_LIBRT + struct timespec m_new; + clock_gettime(CLOCK_REALTIME, &m_new); + + return ( (double) ( m_new.tv_sec - m_old.tv_sec ) ) + + ( (double) ( m_new.tv_nsec - m_old.tv_nsec ) * 1.0e-9 ); + #else + struct timeval m_new ; + + ::gettimeofday( & m_new , ((struct timezone *) NULL ) ); + + return ( (double) ( m_new.tv_sec - m_old.tv_sec ) ) + + ( (double) ( m_new.tv_usec - m_old.tv_usec ) * 1.0e-6 ); + #endif + } +}; + +} // namespace Impl +} // namespace Kokkos + +#endif /* #ifndef KOKKOS_IMPLWALLTIME_HPP */ + diff --git a/lib/kokkos/core/src/impl/Kokkos_Traits.hpp b/lib/kokkos/core/src/impl/Kokkos_Traits.hpp new file mode 100755 index 0000000000..69bab99965 --- /dev/null +++ b/lib/kokkos/core/src/impl/Kokkos_Traits.hpp @@ -0,0 +1,370 @@ +/* +//@HEADER +// ************************************************************************ +// +// Kokkos: Manycore Performance-Portable Multidimensional Arrays +// Copyright (2012) Sandia Corporation +// +// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +// the U.S. Government retains certain rights in this software. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// 1. Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// 3. Neither the name of the Corporation nor the names of the +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Questions? Contact H. Carter Edwards (hcedwar@sandia.gov) +// +// ************************************************************************ +//@HEADER +*/ + +#ifndef KOKKOSTRAITS_HPP +#define KOKKOSTRAITS_HPP + +#include +#include +#include + +namespace Kokkos { +namespace Impl { + +/* C++11 conformal compile-time type traits utilities. + * Prefer to use C++11 when portably available. + */ +//---------------------------------------------------------------------------- +// C++11 Helpers: + +template < class T , T v > +struct integral_constant +{ + // Declaration of 'static const' causes an unresolved linker symbol in debug + // static const T value = v ; + enum { value = T(v) }; + typedef T value_type; + typedef integral_constant type; + KOKKOS_INLINE_FUNCTION operator T() { return v ; } +}; + +typedef integral_constant false_type ; +typedef integral_constant true_type ; + +//---------------------------------------------------------------------------- +// C++11 Type relationships: + +template< class X , class Y > struct is_same : public false_type {}; +template< class X > struct is_same : public true_type {}; + +//---------------------------------------------------------------------------- +// C++11 Type properties: + +template struct is_const : public false_type {}; +template struct is_const : public true_type {}; +template struct is_const : public true_type {}; + +template struct is_array : public false_type {}; +template struct is_array< T[] > : public true_type {}; +template struct is_array< T[N] > : public true_type {}; + +//---------------------------------------------------------------------------- +// C++11 Type transformations: + +template struct remove_const { typedef T type; }; +template struct remove_const { typedef T type; }; +template struct remove_const { typedef T & type; }; + +template struct add_const { typedef const T type; }; +template struct add_const { typedef const T & type; }; +template struct add_const { typedef const T type; }; +template struct add_const { typedef const T & type; }; + +template struct remove_reference { typedef T type ; }; +template struct remove_reference< T & > { typedef T type ; }; +template struct remove_reference< const T & > { typedef const T type ; }; + +template struct remove_extent { typedef T type ; }; +template struct remove_extent { typedef T type ; }; +template struct remove_extent { typedef T type ; }; + +//---------------------------------------------------------------------------- +// C++11 Other type generators: + +template< bool , class T , class F > +struct condition { typedef F type ; }; + +template< class T , class F > +struct condition { typedef T type ; }; + +template< bool , class = void > +struct enable_if ; + +template< class T > +struct enable_if< true , T > { typedef T type ; }; + +//---------------------------------------------------------------------------- + +} // namespace Impl +} // namespace Kokkos + +//---------------------------------------------------------------------------- +//---------------------------------------------------------------------------- +// Other traits + +namespace Kokkos { +namespace Impl { + +//---------------------------------------------------------------------------- + +template< class , class T = void > +struct enable_if_type { typedef T type ; }; + +//---------------------------------------------------------------------------- + +template< bool B > +struct bool_ : public integral_constant {}; + +template< unsigned I > +struct unsigned_ : public integral_constant {}; + +template< int I > +struct int_ : public integral_constant {}; + +typedef bool_ true_; +typedef bool_ false_; +//---------------------------------------------------------------------------- +// if_ + +template < bool Cond , typename TrueType , typename FalseType> +struct if_c +{ + enum { value = Cond }; + + typedef FalseType type; + + + typedef typename remove_const< + typename remove_reference::type >::type value_type ; + + typedef typename add_const::type const_value_type ; + + static KOKKOS_INLINE_FUNCTION + const_value_type & select( const_value_type & v ) { return v ; } + + static KOKKOS_INLINE_FUNCTION + value_type & select( value_type & v ) { return v ; } + + template< class T > + static KOKKOS_INLINE_FUNCTION + value_type & select( const T & ) { value_type * ptr(0); return *ptr ; } + + + template< class T > + static KOKKOS_INLINE_FUNCTION + const_value_type & select( const T & , const_value_type & v ) { return v ; } + + template< class T > + static KOKKOS_INLINE_FUNCTION + value_type & select( const T & , value_type & v ) { return v ; } +}; + +template +struct if_c< true , TrueType , FalseType > +{ + enum { value = true }; + + typedef TrueType type; + + + typedef typename remove_const< + typename remove_reference::type >::type value_type ; + + typedef typename add_const::type const_value_type ; + + static KOKKOS_INLINE_FUNCTION + const_value_type & select( const_value_type & v ) { return v ; } + + static KOKKOS_INLINE_FUNCTION + value_type & select( value_type & v ) { return v ; } + + template< class T > + static KOKKOS_INLINE_FUNCTION + value_type & select( const T & ) { value_type * ptr(0); return *ptr ; } + + + template< class F > + static KOKKOS_INLINE_FUNCTION + const_value_type & select( const_value_type & v , const F & ) { return v ; } + + template< class F > + static KOKKOS_INLINE_FUNCTION + value_type & select( value_type & v , const F & ) { return v ; } +}; + +template< typename TrueType > +struct if_c< false , TrueType , void > +{ + enum { value = false }; + + typedef void type ; + typedef void value_type ; +}; + +template< typename FalseType > +struct if_c< true , void , FalseType > +{ + enum { value = true }; + + typedef void type ; + typedef void value_type ; +}; + +template +struct if_ : public if_c {}; + +//---------------------------------------------------------------------------- + +// Allows aliased types: +template< typename T > +struct is_integral : public integral_constant< bool , + ( + Impl::is_same< T , char >::value || + Impl::is_same< T , unsigned char >::value || + Impl::is_same< T , short int >::value || + Impl::is_same< T , unsigned short int >::value || + Impl::is_same< T , int >::value || + Impl::is_same< T , unsigned int >::value || + Impl::is_same< T , long int >::value || + Impl::is_same< T , unsigned long int >::value || + Impl::is_same< T , long long int >::value || + Impl::is_same< T , unsigned long long int >::value || + + Impl::is_same< T , int8_t >::value || + Impl::is_same< T , int16_t >::value || + Impl::is_same< T , int32_t >::value || + Impl::is_same< T , int64_t >::value || + Impl::is_same< T , uint8_t >::value || + Impl::is_same< T , uint16_t >::value || + Impl::is_same< T , uint32_t >::value || + Impl::is_same< T , uint64_t >::value + )> +{}; + +//---------------------------------------------------------------------------- + + +template < size_t N > +struct is_power_of_two +{ + enum type { value = (N > 0) && !(N & (N-1)) }; +}; + +template < size_t N , bool OK = is_power_of_two::value > +struct power_of_two ; + +template < size_t N > +struct power_of_two +{ + enum type { value = 1+ power_of_two<(N>>1),true>::value }; +}; + +template <> +struct power_of_two<2,true> +{ + enum type { value = 1 }; +}; + +template <> +struct power_of_two<1,true> +{ + enum type { value = 0 }; +}; + +/** \brief If power of two then return power, + * otherwise return ~0u. + */ +static KOKKOS_FORCEINLINE_FUNCTION +unsigned power_of_two_if_valid( const unsigned N ) +{ + unsigned p = ~0u ; + if ( N && ! ( N & ( N - 1 ) ) ) { +#if defined( __CUDA_ARCH__ ) + p = __ffs(N) - 1 ; +#elif defined( __GNUC__ ) || defined( __GNUG__ ) + p = __builtin_ffs(N) - 1 ; +#elif defined( __INTEL_COMPILER ) + p = _bit_scan_forward(N); +#else + p = 0 ; + for ( unsigned j = 1 ; ! ( N & j ) ; j <<= 1 ) { ++p ; } +#endif + } + return p ; +} + +//---------------------------------------------------------------------------- + +template< typename T , T v , bool NonZero = ( v != T(0) ) > +struct integral_nonzero_constant +{ + // Declaration of 'static const' causes an unresolved linker symbol in debug + // static const T value = v ; + enum { value = T(v) }; + typedef T value_type ; + typedef integral_nonzero_constant type ; + KOKKOS_INLINE_FUNCTION integral_nonzero_constant( const T & ) {} +}; + +template< typename T , T zero > +struct integral_nonzero_constant +{ + const T value ; + typedef T value_type ; + typedef integral_nonzero_constant type ; + KOKKOS_INLINE_FUNCTION integral_nonzero_constant( const T & v ) : value(v) {} +}; + +//---------------------------------------------------------------------------- + +template < class C > struct is_integral_constant : public false_ +{ + typedef void integral_type ; + enum { integral_value = 0 }; +}; + +template < typename T , T v > +struct is_integral_constant< integral_constant > : public true_ +{ + typedef T integral_type ; + enum { integral_value = v }; +}; + +} // namespace Impl +} // namespace Kokkos + +//---------------------------------------------------------------------------- +//---------------------------------------------------------------------------- + +#endif /* #ifndef KOKKOSTRAITS_HPP */ + diff --git a/lib/kokkos/core/src/impl/Kokkos_ViewDefault.hpp b/lib/kokkos/core/src/impl/Kokkos_ViewDefault.hpp new file mode 100755 index 0000000000..75b893bef2 --- /dev/null +++ b/lib/kokkos/core/src/impl/Kokkos_ViewDefault.hpp @@ -0,0 +1,2818 @@ +/* +//@HEADER +// ************************************************************************ +// +// Kokkos: Manycore Performance-Portable Multidimensional Arrays +// Copyright (2012) Sandia Corporation +// +// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +// the U.S. Government retains certain rights in this software. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// 1. Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// 3. Neither the name of the Corporation nor the names of the +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Questions? Contact H. Carter Edwards (hcedwar@sandia.gov) +// +// ************************************************************************ +//@HEADER +*/ + +#ifndef KOKKOS_VIEWDEFAULT_HPP +#define KOKKOS_VIEWDEFAULT_HPP + +//---------------------------------------------------------------------------- +//---------------------------------------------------------------------------- + +namespace Kokkos { +namespace Impl { + +template<> +struct ViewAssignment< ViewDefault , ViewDefault , void > +{ + typedef ViewDefault Specialize ; + + //------------------------------------ + /** \brief Compatible value and shape */ + + template< class DT , class DL , class DD , class DM , + class ST , class SL , class SD , class SM > + KOKKOS_INLINE_FUNCTION + ViewAssignment( View & dst , + const View & src , + const typename enable_if<( + ViewAssignable< ViewTraits , + ViewTraits >::value + || + ( ViewAssignable< ViewTraits , + ViewTraits >::assignable_value + && + ShapeCompatible< typename ViewTraits::shape_type , + typename ViewTraits::shape_type >::value + && + is_same< typename ViewTraits::array_layout,LayoutStride>::value ) + )>::type * = 0 ) + { + dst.m_management.decrement( dst.m_ptr_on_device ); + + dst.m_offset_map.assign( src.m_offset_map ); + + dst.m_management = src.m_management ; + + dst.m_ptr_on_device = typename ViewDataManagement< ViewTraits >::handle_type( src.m_ptr_on_device ); + + dst.m_management.increment( dst.m_ptr_on_device ); + } + + //------------------------------------ + /** \brief Extract Rank-0 from Rank-1 */ + + template< class DT , class DL , class DD , class DM , + class ST , class SL , class SD , class SM > + KOKKOS_INLINE_FUNCTION + ViewAssignment( View & dst , + const View & src , + const typename enable_if< ( + ViewAssignable< ViewTraits , + ViewTraits >::assignable_value && + ( ViewTraits::rank == 0 ) && + ( ViewTraits::rank == 1 ) + ), unsigned >::type i0 ) + { + assert_shape_bounds( src.m_offset_map , 1 , i0 ); + + dst.m_management.decrement( dst.m_ptr_on_device ); + + dst.m_management = src.m_management ; + + dst.m_ptr_on_device = src.ptr_on_device() + i0 ; + + dst.m_management.increment( dst.m_ptr_on_device ); + } + + //------------------------------------ + /** \brief Extract Rank-0 from Rank-2 */ + + template< class DT , class DL , class DD , class DM , + class ST , class SL , class SD , class SM > + KOKKOS_INLINE_FUNCTION + ViewAssignment( View & dst , + const View & src , + const typename enable_if< ( + ViewAssignable< ViewTraits , + ViewTraits >::assignable_value && + ( ViewTraits::rank == 0 ) && + ( ViewTraits::rank == 2 ) + ), unsigned >::type i0 , + const unsigned i1 ) + { + assert_shape_bounds( src.m_offset_map , 2 , i0 , i1 ); + + dst.m_management.decrement( dst.m_ptr_on_device ); + + dst.m_management = src.m_management ; + + dst.m_ptr_on_device = src.ptr_on_device() + src.m_offset_map(i0,i1); + + dst.m_management.increment( dst.m_ptr_on_device ); + } + + //------------------------------------ + /** \brief Extract Rank-0 from Rank-3 */ + + template< class DT , class DL , class DD , class DM , + class ST , class SL , class SD , class SM > + KOKKOS_INLINE_FUNCTION + ViewAssignment( View & dst , + const View & src , + const typename enable_if< ( + ViewAssignable< ViewTraits , + ViewTraits >::assignable_value && + ( ViewTraits::rank == 0 ) && + ( ViewTraits::rank == 3 ) + ), unsigned >::type i0 , + const unsigned i1 , + const unsigned i2 ) + { + assert_shape_bounds( src.m_offset_map, 3, i0, i1, i2 ); + + dst.m_management.decrement( dst.m_ptr_on_device ); + + dst.m_management = src.m_management ; + + dst.m_ptr_on_device = src.ptr_on_device() + src.m_offset_map(i0,i1,i2); + + dst.m_management.increment( dst.m_ptr_on_device ); + } + + //------------------------------------ + /** \brief Extract Rank-0 from Rank-4 */ + + template< class DT , class DL , class DD , class DM , + class ST , class SL , class SD , class SM > + KOKKOS_INLINE_FUNCTION + ViewAssignment( View & dst , + const View & src , + const typename enable_if< ( + ViewAssignable< ViewTraits , + ViewTraits >::assignable_value && + ( ViewTraits::rank == 0 ) && + ( ViewTraits::rank == 4 ) + ), unsigned >::type i0 , + const unsigned i1 , + const unsigned i2 , + const unsigned i3 ) + { + assert_shape_bounds( src.m_offset_map, 4, i0, i1, i2, i3 ); + + dst.m_management.decrement( dst.m_ptr_on_device ); + + dst.m_management = src.m_management ; + + dst.m_ptr_on_device = src.ptr_on_device() + src.m_offset_map(i0,i1,i2,i3); + + dst.m_management.increment( dst.m_ptr_on_device ); + } + + //------------------------------------ + /** \brief Extract Rank-0 from Rank-5 */ + + template< class DT , class DL , class DD , class DM , + class ST , class SL , class SD , class SM > + KOKKOS_INLINE_FUNCTION + ViewAssignment( View & dst , + const View & src , + const typename enable_if< ( + ViewAssignable< ViewTraits , + ViewTraits >::assignable_value && + ( ViewTraits::rank == 0 ) && + ( ViewTraits::rank == 5 ) + ), unsigned >::type i0 , + const unsigned i1 , + const unsigned i2 , + const unsigned i3 , + const unsigned i4 ) + { + assert_shape_bounds( src.m_offset_map, 5, i0, i1, i2, i3, i4); + + dst.m_management.decrement( dst.m_ptr_on_device ); + + dst.m_management = src.m_management ; + + dst.m_ptr_on_device = src.ptr_on_device() + src.m_offset_map(i0,i1,i2,i3,i4); + + dst.m_management.increment( dst.m_ptr_on_device ); + } + + //------------------------------------ + /** \brief Extract Rank-0 from Rank-6 */ + + template< class DT , class DL , class DD , class DM , + class ST , class SL , class SD , class SM > + KOKKOS_INLINE_FUNCTION + ViewAssignment( View & dst , + const View & src , + const typename enable_if< ( + ViewAssignable< ViewTraits , + ViewTraits >::assignable_value && + ( ViewTraits::rank == 0 ) && + ( ViewTraits::rank == 6 ) + ), unsigned >::type i0 , + const unsigned i1 , + const unsigned i2 , + const unsigned i3 , + const unsigned i4 , + const unsigned i5 ) + { + assert_shape_bounds( src.m_offset_map, 6, i0, i1, i2, i3, i4, i5); + + dst.m_management.decrement( dst.m_ptr_on_device ); + + dst.m_management = src.m_management ; + + dst.m_ptr_on_device = src.ptr_on_device() + src.m_offset_map(i0,i1,i2,i3,i4,i5); + + dst.m_management.increment( dst.m_ptr_on_device ); + } + + //------------------------------------ + /** \brief Extract Rank-0 from Rank-7 */ + + template< class DT , class DL , class DD , class DM , + class ST , class SL , class SD , class SM > + KOKKOS_INLINE_FUNCTION + ViewAssignment( View & dst , + const View & src , + const typename enable_if< ( + ViewAssignable< ViewTraits , + ViewTraits >::assignable_value && + ( ViewTraits::rank == 0 ) && + ( ViewTraits::rank == 7 ) + ), unsigned >::type i0 , + const unsigned i1 , + const unsigned i2 , + const unsigned i3 , + const unsigned i4 , + const unsigned i5 , + const unsigned i6 ) + { + assert_shape_bounds( src.m_offset_map, 7, i0, i1, i2, i3, i4, i5, i6 ); + + dst.m_management.decrement( dst.m_ptr_on_device ); + + dst.m_management = src.m_management ; + + dst.m_ptr_on_device = src.ptr_on_device() + src.m_offset_map(i0,i1,i2,i3,i4,i5,i6); + + dst.m_management.increment( dst.m_ptr_on_device ); + } + + //------------------------------------ + /** \brief Extract Rank-0 from Rank-8 */ + + template< class DT , class DL , class DD , class DM , + class ST , class SL , class SD , class SM > + KOKKOS_INLINE_FUNCTION + ViewAssignment( View & dst , + const View & src , + const typename enable_if< ( + ViewAssignable< ViewTraits , + ViewTraits >::assignable_value && + ( ViewTraits::rank == 0 ) && + ( ViewTraits::rank == 8 ) + ), unsigned >::type i0 , + const unsigned i1 , + const unsigned i2 , + const unsigned i3 , + const unsigned i4 , + const unsigned i5 , + const unsigned i6 , + const unsigned i7 ) + { + assert_shape_bounds( src.m_offset_map, 8, i0, i1, i2, i3, i4, i5, i6, i7 ); + + dst.m_management.decrement( dst.m_ptr_on_device ); + + dst.m_management = src.m_management ; + + dst.m_ptr_on_device = src.ptr_on_device() + src.m_offset_map(i0,i1,i2,i3,i4,i5,i6,i7); + + dst.m_management.increment( dst.m_ptr_on_device ); + } + + //------------------------------------ + /** \brief Extract Rank-1 array from range of Rank-1 array, either layout */ + template< class DT , class DL , class DD , class DM , + class ST , class SL , class SD , class SM , + typename iType > + KOKKOS_INLINE_FUNCTION + ViewAssignment( View & dst , + const View & src , + const std::pair & range , + typename enable_if< ( + ViewAssignable< ViewTraits , ViewTraits >::assignable_value + && + ( ViewTraits::rank == 1 ) + && + ( ViewTraits::rank == 1 ) + && + ( ViewTraits::rank_dynamic == 1 ) + ) >::type * = 0 ) + { + dst.m_management.decrement( dst.m_ptr_on_device ); + + dst.m_offset_map.N0 = 0 ; + dst.m_ptr_on_device = 0 ; + + if ( range.first < range.second ) { + assert_shape_bounds( src.m_offset_map , 1 , range.first ); + assert_shape_bounds( src.m_offset_map , 1 , range.second - 1 ); + + dst.m_management = src.m_management ; + dst.m_offset_map.N0 = range.second - range.first ; + dst.m_ptr_on_device = src.ptr_on_device() + range.first ; + + dst.m_management.increment( dst.m_ptr_on_device ); + } + } + + //------------------------------------ + /** \brief Extract Rank-1 array from LayoutLeft Rank-2 array, using ALL as first argument. */ + template< class DT , class DL , class DD , class DM , + class ST , class SL , class SD , class SM > + KOKKOS_INLINE_FUNCTION + ViewAssignment( View & dst , + const View & src , + const ALL & , + const typename enable_if< ( + ViewAssignable< ViewTraits , ViewTraits >::assignable_value + && + is_same< typename ViewTraits::array_layout , LayoutLeft >::value + && + ( ViewTraits::rank == 2 ) + && + ( ViewTraits::rank == 1 ) + && + ( ViewTraits::rank_dynamic == 1 ) + ), unsigned >::type i1 ) + { + dst.m_management.decrement( dst.m_ptr_on_device ); + + dst.m_management = src.m_management ; + dst.m_offset_map.N0 = src.m_offset_map.N0 ; + dst.m_ptr_on_device = src.ptr_on_device() + src.m_offset_map(0,i1); + + dst.m_management.increment( dst.m_ptr_on_device ); + } + + + //------------------------------------ + /** \brief Extract Rank-1 array from LayoutLeft Rank-2 array, using a row range as first argument. */ + template< class DT , class DL , class DD , class DM , + class ST , class SL , class SD , class SM , + typename IndexType > + KOKKOS_INLINE_FUNCTION + ViewAssignment( View & dst , + const View & src , + const std::pair& rowRange, + const typename enable_if< ( + ViewAssignable< ViewTraits , ViewTraits >::assignable_value + && + is_same< typename ViewTraits::array_layout , LayoutLeft >::value + && + ( ViewTraits::rank == 2 ) + && + ( ViewTraits::rank == 1 ) + && + ( ViewTraits::rank_dynamic == 1 ) + ), IndexType >::type columnIndex ) + { + dst.m_management.decrement( dst.m_ptr_on_device ); + + if (rowRange.first < rowRange.second) { // valid row range + dst.m_management = src.m_management; + dst.m_offset_map.N0 = rowRange.second - rowRange.first; + dst.m_ptr_on_device = src.ptr_on_device () + + src.m_offset_map (rowRange.first, columnIndex); + + dst.m_management.increment( dst.m_ptr_on_device ); + } + else { // not a valid row range + dst.m_offset_map.N0 = 0; + dst.m_ptr_on_device = 0; + } + } + + + //------------------------------------ + /** \brief Extract Rank-1 array from LayoutRight Rank-2 array. */ + template< class DT , class DL , class DD , class DM , + class ST , class SL , class SD , class SM > + KOKKOS_INLINE_FUNCTION + ViewAssignment( View & dst , + const View & src , + const unsigned i0 , + const typename enable_if< ( + ViewAssignable< ViewTraits , ViewTraits >::assignable_value + && + is_same< typename ViewTraits::array_layout , LayoutRight >::value + && + ( ViewTraits::rank == 2 ) + && + ( ViewTraits::rank == 1 ) + && + ( ViewTraits::rank_dynamic == 1 ) + ), ALL >::type & ) + { + dst.m_management.decrement( dst.m_ptr_on_device ); + + dst.m_management = src.m_management ; + dst.m_offset_map.N0 = src.m_offset_map.N1 ; + dst.m_ptr_on_device = src.ptr_on_device() + src.m_offset_map(i0,0); + + dst.m_management.increment( dst.m_ptr_on_device ); + } + + //------------------------------------ + /** \brief Extract Rank-2 array from LayoutLeft Rank-2 array. */ + template< class DT , class DL , class DD , class DM , + class ST , class SL , class SD , class SM , + typename iType > + KOKKOS_INLINE_FUNCTION + ViewAssignment( View & dst , + const View & src , + const std::pair & range , + const typename enable_if< ( + ViewAssignable< ViewTraits , ViewTraits >::assignable_value + && + is_same< typename ViewTraits::array_layout , LayoutLeft >::value + && + ( ViewTraits::rank == 2 ) + && + ( ViewTraits::rank == 2 ) + && + ( ViewTraits::rank_dynamic == 2 ) + ), unsigned >::type i1 ) + { + assert_shape_bounds( src.m_offset_map , 2 , range.first , i1 ); + assert_shape_bounds( src.m_offset_map , 2 , range.second - 1 , i1 ); + + dst.m_management.decrement( dst.m_ptr_on_device ); + + if ( range.first < range.second ) { + dst.m_management = src.m_management ; + dst.m_offset_map.N0 = range.second - range.first ; + dst.m_offset_map.N1 = 1 ; + dst.m_offset_map.S0 = range.second - range.first ; + dst.m_ptr_on_device = src.ptr_on_device() + src.m_offset_map(range.first,i1); + + dst.m_management.increment( dst.m_ptr_on_device ); + } + } + + //------------------------------------ + /** \brief Extract Rank-2 array from LayoutLeft Rank-2 array. */ + template< class DT , class DL , class DD , class DM , + class ST , class SL , class SD , class SM > + KOKKOS_INLINE_FUNCTION + ViewAssignment( View & dst , + const View & src , + const ALL & , + const typename enable_if< ( + ViewAssignable< ViewTraits , ViewTraits >::assignable_value + && + is_same< typename ViewTraits::array_layout , LayoutLeft >::value + && + ( ViewTraits::rank == 2 ) + && + ( ViewTraits::rank == 2 ) + && + ( ViewTraits::rank_dynamic == 2 ) + ), unsigned >::type i1 ) + { + dst.m_management.decrement( dst.m_ptr_on_device ); + + dst.m_management = src.m_management ; + dst.m_offset_map.N0 = src.m_offset_map.N0 ; + dst.m_offset_map.N1 = 1 ; + + dst.m_offset_map.S0 = src.m_offset_map.N0 ; + dst.m_ptr_on_device = src.ptr_on_device() + src.m_offset_map(0,i1); + + dst.m_management.increment( dst.m_ptr_on_device ); + } + + //------------------------------------ + /** \brief Extract Rank-2 array from LayoutRight Rank-2 array. */ + template< class DT , class DL , class DD , class DM , + class ST , class SL , class SD , class SM > + KOKKOS_INLINE_FUNCTION + ViewAssignment( View & dst , + const View & src , + const unsigned i0 , + const typename enable_if< ( + ViewAssignable< ViewTraits , ViewTraits >::assignable_value + && + is_same< typename ViewTraits::array_layout , LayoutRight >::value + && + ( ViewTraits::rank == 2 ) + && + ( ViewTraits::rank == 2 ) + && + ( ViewTraits::rank_dynamic == 2 ) + ), ALL >::type & ) + { + dst.m_management.decrement( dst.m_ptr_on_device ); + + dst.m_management = src.m_management ; + dst.m_offset_map.N0 = 1 ; + dst.m_offset_map.N1 = src.m_offset_map.N1 ; + dst.m_offset_map.SR = src.m_offset_map.SR ; + dst.m_ptr_on_device = src.ptr_on_device() + src.m_offset_map(i0,0); + + dst.m_management.increment( dst.m_ptr_on_device ); + } + //------------------------------------ + /** \brief Extract LayoutRight Rank-N array from range of LayoutRight Rank-N array */ + template< class DT , class DL , class DD , class DM , + class ST , class SL , class SD , class SM , + typename iType > + KOKKOS_INLINE_FUNCTION + ViewAssignment( View & dst , + const View & src , + const std::pair & range , + typename enable_if< ( + ViewAssignable< ViewTraits , ViewTraits >::value + && + Impl::is_same< typename ViewTraits::array_layout , LayoutRight >::value + && + ( ViewTraits::rank > 1 ) + && + ( ViewTraits::rank_dynamic > 0 ) + )>::type * = 0 ) + { + //typedef ViewTraits traits_type ; // unused + //typedef typename traits_type::shape_type shape_type ; // unused + //typedef typename View::stride_type stride_type ; // unused + + dst.m_management.decrement( dst.m_ptr_on_device ); + + dst.m_offset_map.assign( 0, 0, 0, 0, 0, 0, 0, 0 ); + + dst.m_ptr_on_device = 0 ; + + if ( ( range.first == range.second ) || + ( (src.capacity()==0u) && (range.second + KOKKOS_INLINE_FUNCTION + ViewAssignment( View & dst , + const View & src , + const std::pair & range0 , + const std::pair & range1 , + typename enable_if< ( + ViewAssignable< ViewTraits , ViewTraits >::value + && + ViewTraits::rank == 2 + && + ViewTraits::rank_dynamic == 2 + ) >::type * = 0 ) + { + dst.m_management.decrement( dst.m_ptr_on_device ); + dst.m_offset_map.assign(0,0,0,0, 0,0,0,0); + dst.m_ptr_on_device = 0 ; + + if ( (range0.first == range0.second) || + (range1.first == range1.second) || + ( ( src.capacity() == 0u ) && + ( long(range0.second) < long(src.m_offset_map.N0) ) && + ( long(range1.second) < long(src.m_offset_map.N1) ) ) ) { + + dst.m_offset_map.assign( src.m_offset_map ); + dst.m_offset_map.N0 = range0.second - range0.first ; + dst.m_offset_map.N1 = range1.second - range1.first ; + } + else if ( (range0.first < range0.second && range1.first < range1.second) ) { + + assert_shape_bounds( src.m_offset_map , 2 , range0.first , range1.first ); + assert_shape_bounds( src.m_offset_map , 2 , range0.second - 1 , range1.second - 1 ); + + dst.m_offset_map.assign( src.m_offset_map ); + dst.m_offset_map.N0 = range0.second - range0.first ; + dst.m_offset_map.N1 = range1.second - range1.first ; + + dst.m_management = src.m_management ; + + dst.m_ptr_on_device = src.ptr_on_device() + src.m_offset_map(range0.first,range1.first); + + dst.m_management.increment( dst.m_ptr_on_device ); + } + } + + //------------------------------------ + /** \brief Extract rank-2 from rank-2 array */ + template< class DT , class DL , class DD , class DM , + class ST , class SL , class SD , class SM , + typename iType > + KOKKOS_INLINE_FUNCTION + ViewAssignment( View & dst , + const View & src , + ALL , + const std::pair & range1 , + typename enable_if< ( + ViewAssignable< ViewTraits , ViewTraits >::value + && + ViewTraits::rank == 2 + && + ViewTraits::rank_dynamic == 2 + ) >::type * = 0 ) + { + dst.m_management.decrement( dst.m_ptr_on_device ); + dst.m_offset_map.assign(0,0,0,0, 0,0,0,0); + dst.m_ptr_on_device = 0 ; + + if ( (range1.first == range1.second) || ( (src.capacity()==0) && (range1.second + KOKKOS_INLINE_FUNCTION + ViewAssignment( View & dst , + const View & src , + const std::pair & range0 , + ALL , + typename enable_if< ( + ViewAssignable< ViewTraits , ViewTraits >::value + && + ViewTraits::rank == 2 + && + ViewTraits::rank_dynamic == 2 + ) >::type * = 0 ) + { + dst.m_management.decrement( dst.m_ptr_on_device ); + dst.m_offset_map.assign(0,0,0,0, 0,0,0,0); + dst.m_ptr_on_device = 0 ; + + if ( (range0.first == range0.second) || ( (src.capacity()==0) && (range0.second + KOKKOS_INLINE_FUNCTION + ViewAssignment( View & dst , + const View & src , + const std::pair & range0 , + ALL , + typename enable_if< ( + ViewAssignable< ViewTraits , ViewTraits >::value + && + ViewTraits::rank == 2 + && + ViewTraits::rank_dynamic == 1 + ) >::type * = 0 ) + { + dst.m_tracking.decrement( dst.ptr_on_device() ); + dst.m_offset_map.assign(0,0,0,0, 0,0,0,0); + dst.m_ptr_on_device = 0 ; + + if ( (range0.first == range0.second) || ( (src.capacity()==0) && (range0.second + KOKKOS_INLINE_FUNCTION + ViewAssignment( View & dst , + const View & src , + const unsigned i0 , + const ALL & , + const typename enable_if< ( + ViewAssignable< ViewTraits , ViewTraits >::assignable_value + && + is_same< typename ViewTraits::array_layout , LayoutRight >::value + && + ( ViewTraits::rank == 3 ) + && + ( ViewTraits::rank == 2 ) + && + ( ViewTraits::rank_dynamic == 2 ) + ), ALL >::type & ) + { + //typedef ViewTraits traits_type ; // unused + + dst.m_management.decrement( dst.m_ptr_on_device ); + + dst.m_management = src.m_management ; + dst.m_offset_map.N0 = src.m_offset_map.N1 ; + dst.m_offset_map.N1 = src.m_offset_map.N2 ; + dst.m_offset_map.SR = dst.m_offset_map.N1 ; + dst.m_ptr_on_device = &src(i0,0,0); + + dst.m_management.increment( dst.m_ptr_on_device ); + } + + //------------------------------------ + /** \brief Extract Rank-2 array from LayoutRight Rank-4 array. */ + template< class DT , class DL , class DD , class DM , + class ST , class SL , class SD , class SM > + KOKKOS_INLINE_FUNCTION + ViewAssignment( View & dst , + const View & src , + const unsigned i0 , + const unsigned i1 , + const ALL & , + const typename enable_if< ( + ViewAssignable< ViewTraits , ViewTraits >::assignable_value + && + is_same< typename ViewTraits::array_layout , LayoutRight >::value + && + ( ViewTraits::rank == 4 ) + && + ( ViewTraits::rank == 2 ) + && + ( ViewTraits::rank_dynamic == 2 ) + ), ALL >::type & ) + { + //typedef ViewTraits traits_type ; // unused + + dst.m_management.decrement( dst.m_ptr_on_device ); + + dst.m_management = src.m_management ; + dst.m_offset_map.N0 = src.m_offset_map.N2 ; + dst.m_offset_map.N1 = src.m_offset_map.N3 ; + dst.m_offset_map.SR = dst.m_offset_map.N1 ; + dst.m_ptr_on_device = &src(i0,i1,0,0); + + dst.m_management.increment( dst.m_ptr_on_device ); + } + + //------------------------------------ + /** \brief Extract Rank-2 array from LayoutRight Rank-5 array. */ + template< class DT , class DL , class DD , class DM , + class ST , class SL , class SD , class SM > + KOKKOS_INLINE_FUNCTION + ViewAssignment( View & dst , + const View & src , + const unsigned i0 , + const unsigned i1 , + const unsigned i2 , + const ALL & , + const typename enable_if< ( + ViewAssignable< ViewTraits , ViewTraits >::assignable_value + && + is_same< typename ViewTraits::array_layout , LayoutRight >::value + && + ( ViewTraits::rank == 5 ) + && + ( ViewTraits::rank == 2 ) + && + ( ViewTraits::rank_dynamic == 2 ) + ), ALL >::type & ) + { + //typedef ViewTraits traits_type ; // unused + + dst.m_management.decrement( dst.m_ptr_on_device ); + + dst.m_management = src.m_management ; + dst.m_offset_map.N0 = src.m_offset_map.N3 ; + dst.m_offset_map.N1 = src.m_offset_map.N4 ; + dst.m_offset_map.SR = dst.m_offset_map.N1 ; + dst.m_ptr_on_device = &src(i0,i1,i2,0,0); + + dst.m_management.increment( dst.m_ptr_on_device ); + } + + + //------------------------------------ + /** \brief Extract Rank-2 array from LayoutRight Rank-6 array. */ + template< class DT , class DL , class DD , class DM , + class ST , class SL , class SD , class SM > + KOKKOS_INLINE_FUNCTION + ViewAssignment( View & dst , + const View & src , + const unsigned i0 , + const unsigned i1 , + const unsigned i2 , + const unsigned i3 , + const ALL & , + const typename enable_if< ( + ViewAssignable< ViewTraits , ViewTraits >::assignable_value + && + is_same< typename ViewTraits::array_layout , LayoutRight >::value + && + ( ViewTraits::rank == 6 ) + && + ( ViewTraits::rank == 2 ) + && + ( ViewTraits::rank_dynamic == 2 ) + ), ALL >::type & ) + { + //typedef ViewTraits traits_type ; // unused + + dst.m_management.decrement( dst.m_ptr_on_device ); + + dst.m_management = src.m_management ; + dst.m_offset_map.N0 = src.m_offset_map.N4 ; + dst.m_offset_map.N1 = src.m_offset_map.N5 ; + dst.m_offset_map.SR = dst.m_offset_map.N1 ; + dst.m_ptr_on_device = &src(i0,i1,i2,i3,0,0); + + dst.m_management.increment( dst.m_ptr_on_device ); + } + + //------------------------------------ + /** \brief Extract Rank-2 array from LayoutRight Rank-7 array. */ + template< class DT , class DL , class DD , class DM , + class ST , class SL , class SD , class SM > + KOKKOS_INLINE_FUNCTION + ViewAssignment( View & dst , + const View & src , + const unsigned i0 , + const unsigned i1 , + const unsigned i2 , + const unsigned i3 , + const unsigned i4 , + const ALL & , + const typename enable_if< ( + ViewAssignable< ViewTraits , ViewTraits >::assignable_value + && + is_same< typename ViewTraits::array_layout , LayoutRight >::value + && + ( ViewTraits::rank == 7 ) + && + ( ViewTraits::rank == 2 ) + && + ( ViewTraits::rank_dynamic == 2 ) + ), ALL >::type & ) + { + //typedef ViewTraits traits_type ; // unused + + dst.m_management.decrement( dst.m_ptr_on_device ); + + dst.m_management = src.m_management ; + dst.m_offset_map.N0 = src.m_offset_map.N5 ; + dst.m_offset_map.N1 = src.m_offset_map.N6 ; + dst.m_offset_map.SR = dst.m_offset_map.N1 ; + dst.m_ptr_on_device = &src(i0,i1,i2,i3,i4,0,0); + + dst.m_management.increment( dst.m_ptr_on_device ); + } + + + //------------------------------------ + /** \brief Extract Rank-2 array from LayoutRight Rank-8 array. */ + template< class DT , class DL , class DD , class DM , + class ST , class SL , class SD , class SM > + KOKKOS_INLINE_FUNCTION + ViewAssignment( View & dst , + const View & src , + const unsigned i0 , + const unsigned i1 , + const unsigned i2 , + const unsigned i3 , + const unsigned i4 , + const unsigned i5 , + const ALL & , + const typename enable_if< ( + ViewAssignable< ViewTraits , ViewTraits >::assignable_value + && + is_same< typename ViewTraits::array_layout , LayoutRight >::value + && + ( ViewTraits::rank == 8 ) + && + ( ViewTraits::rank == 2 ) + && + ( ViewTraits::rank_dynamic == 2 ) + ), ALL >::type & ) + { + //typedef ViewTraits traits_type ; // unused + + dst.m_management.decrement( dst.m_ptr_on_device ); + + dst.m_management = src.m_management ; + dst.m_offset_map.N0 = src.m_offset_map.N6 ; + dst.m_offset_map.N1 = src.m_offset_map.N7 ; + dst.m_offset_map.SR = dst.m_offset_map.N1 ; + dst.m_ptr_on_device = &src(i0,i1,i2,i3,i4,i5,0,0); + + dst.m_management.increment( dst.m_ptr_on_device ); + } + + //------------------------------------ + /** \brief Extract Rank-3 array from LayoutRight Rank-4 array. */ + template< class DT , class DL , class DD , class DM , + class ST , class SL , class SD , class SM > + KOKKOS_INLINE_FUNCTION + ViewAssignment( View & dst , + const View & src , + const unsigned i0 , + const ALL & , + const ALL & , + const typename enable_if< ( + ViewAssignable< ViewTraits , ViewTraits >::assignable_value + && + is_same< typename ViewTraits::array_layout , LayoutRight >::value + && + ( ViewTraits::rank == 4 ) + && + ( ViewTraits::rank == 3 ) + && + ( ViewTraits::rank_dynamic == 3 ) + ), ALL >::type & ) + { + //typedef ViewTraits traits_type ; // unused + + dst.m_management.decrement( dst.m_ptr_on_device ); + + dst.m_management = src.m_management ; + dst.m_offset_map.N0 = src.m_offset_map.N1 ; + dst.m_offset_map.N1 = src.m_offset_map.N2 ; + dst.m_offset_map.N2 = src.m_offset_map.N3 ; + dst.m_offset_map.SR = dst.m_offset_map.N1 * dst.m_offset_map.N2 ; + dst.m_ptr_on_device = &src(i0,0,0,0); + + dst.m_management.increment( dst.m_ptr_on_device ); + } + + //------------------------------------ + /** \brief Extract Rank-3 array from LayoutRight Rank-5 array. */ + template< class DT , class DL , class DD , class DM , + class ST , class SL , class SD , class SM > + KOKKOS_INLINE_FUNCTION + ViewAssignment( View & dst , + const View & src , + const unsigned i0 , + const unsigned i1 , + const ALL & , + const ALL & , + const typename enable_if< ( + ViewAssignable< ViewTraits , ViewTraits >::assignable_value + && + is_same< typename ViewTraits::array_layout , LayoutRight >::value + && + ( ViewTraits::rank == 5 ) + && + ( ViewTraits::rank == 3 ) + && + ( ViewTraits::rank_dynamic == 3 ) + ), ALL >::type & ) + { + //typedef ViewTraits traits_type ; // unused + + dst.m_management.decrement( dst.m_ptr_on_device ); + + dst.m_management = src.m_management ; + dst.m_offset_map.N0 = src.m_offset_map.N2 ; + dst.m_offset_map.N1 = src.m_offset_map.N3 ; + dst.m_offset_map.N2 = src.m_offset_map.N4 ; + dst.m_offset_map.SR = dst.m_offset_map.N1 * dst.m_offset_map.N2 ; + dst.m_ptr_on_device = &src(i0,i1,0,0,0); + + dst.m_management.increment( dst.m_ptr_on_device ); + } + + //------------------------------------ + /** \brief Extract Rank-3 array from LayoutRight Rank-6 array. */ + template< class DT , class DL , class DD , class DM , + class ST , class SL , class SD , class SM > + KOKKOS_INLINE_FUNCTION + ViewAssignment( View & dst , + const View & src , + const unsigned i0 , + const unsigned i1 , + const unsigned i2 , + const ALL & , + const ALL & , + const typename enable_if< ( + ViewAssignable< ViewTraits , ViewTraits >::assignable_value + && + is_same< typename ViewTraits::array_layout , LayoutRight >::value + && + ( ViewTraits::rank == 6 ) + && + ( ViewTraits::rank == 3 ) + && + ( ViewTraits::rank_dynamic == 3 ) + ), ALL >::type & ) + { + //typedef ViewTraits traits_type ; // unused + + dst.m_management.decrement( dst.m_ptr_on_device ); + + dst.m_management = src.m_management ; + dst.m_offset_map.N0 = src.m_offset_map.N3 ; + dst.m_offset_map.N1 = src.m_offset_map.N4 ; + dst.m_offset_map.N2 = src.m_offset_map.N5 ; + dst.m_offset_map.SR = dst.m_offset_map.N1 * dst.m_offset_map.N2 ; + dst.m_ptr_on_device = &src(i0,i1,i2,0,0,0); + + dst.m_management.increment( dst.m_ptr_on_device ); + } + + //------------------------------------ + /** \brief Extract Rank-3 array from LayoutRight Rank-7 array. */ + template< class DT , class DL , class DD , class DM , + class ST , class SL , class SD , class SM > + KOKKOS_INLINE_FUNCTION + ViewAssignment( View & dst , + const View & src , + const unsigned i0 , + const unsigned i1 , + const unsigned i2 , + const unsigned i3 , + const ALL & , + const ALL & , + const typename enable_if< ( + ViewAssignable< ViewTraits , ViewTraits >::assignable_value + && + is_same< typename ViewTraits::array_layout , LayoutRight >::value + && + ( ViewTraits::rank == 7 ) + && + ( ViewTraits::rank == 3 ) + && + ( ViewTraits::rank_dynamic == 3 ) + ), ALL >::type & ) + { + //typedef ViewTraits traits_type ; // unused + + dst.m_management.decrement( dst.m_ptr_on_device ); + + dst.m_management = src.m_management ; + dst.m_offset_map.N0 = src.m_offset_map.N4 ; + dst.m_offset_map.N1 = src.m_offset_map.N5 ; + dst.m_offset_map.N2 = src.m_offset_map.N6 ; + dst.m_offset_map.SR = dst.m_offset_map.N1 * dst.m_offset_map.N2 ; + dst.m_ptr_on_device = &src(i0,i1,i2,i3,0,0,0); + + dst.m_management.increment( dst.m_ptr_on_device ); + } + + //------------------------------------ + /** \brief Extract Rank-3 array from LayoutRight Rank-8 array. */ + template< class DT , class DL , class DD , class DM , + class ST , class SL , class SD , class SM > + KOKKOS_INLINE_FUNCTION + ViewAssignment( View & dst , + const View & src , + const unsigned i0 , + const unsigned i1 , + const unsigned i2 , + const unsigned i3 , + const unsigned i4 , + const ALL & , + const ALL & , + const typename enable_if< ( + ViewAssignable< ViewTraits , ViewTraits >::assignable_value + && + is_same< typename ViewTraits::array_layout , LayoutRight >::value + && + ( ViewTraits::rank == 8 ) + && + ( ViewTraits::rank == 3 ) + && + ( ViewTraits::rank_dynamic == 3 ) + ), ALL >::type & ) + { + //typedef ViewTraits traits_type ; // unused + + dst.m_management.decrement( dst.m_ptr_on_device ); + + dst.m_management = src.m_management ; + dst.m_offset_map.N0 = src.m_offset_map.N5 ; + dst.m_offset_map.N1 = src.m_offset_map.N6 ; + dst.m_offset_map.N2 = src.m_offset_map.N7 ; + dst.m_offset_map.SR = dst.m_offset_map.N1 * dst.m_offset_map.N2 ; + dst.m_ptr_on_device = &src(i0,i1,i2,i3,i4,0,0,0); + + dst.m_management.increment( dst.m_ptr_on_device ); + } + + //------------------------------------ + /** \brief Extract Rank-4 array from LayoutRight Rank-5 array. */ + template< class DT , class DL , class DD , class DM , + class ST , class SL , class SD , class SM > + KOKKOS_INLINE_FUNCTION + ViewAssignment( View & dst , + const View & src , + const unsigned i0 , + const ALL & , + const ALL & , + const ALL & , + const typename enable_if< ( + ViewAssignable< ViewTraits , ViewTraits >::assignable_value + && + is_same< typename ViewTraits::array_layout , LayoutRight >::value + && + ( ViewTraits::rank == 5 ) + && + ( ViewTraits::rank == 4 ) + && + ( ViewTraits::rank_dynamic == 4 ) + ), ALL >::type & ) + { + //typedef ViewTraits traits_type ; // unused + + dst.m_management.decrement( dst.m_ptr_on_device ); + + dst.m_management = src.m_management ; + dst.m_offset_map.N0 = src.m_offset_map.N1 ; + dst.m_offset_map.N1 = src.m_offset_map.N2 ; + dst.m_offset_map.N2 = src.m_offset_map.N3 ; + dst.m_offset_map.N3 = src.m_offset_map.N4 ; + dst.m_offset_map.SR = dst.m_offset_map.N1 * dst.m_offset_map.N2 * + dst.m_offset_map.N3 ; + dst.m_ptr_on_device = &src(i0,0,0,0,0); + + dst.m_management.increment( dst.m_ptr_on_device ); + } + + //------------------------------------ + /** \brief Extract Rank-4 array from LayoutRight Rank-6 array. */ + template< class DT , class DL , class DD , class DM , + class ST , class SL , class SD , class SM > + KOKKOS_INLINE_FUNCTION + ViewAssignment( View & dst , + const View & src , + const unsigned i0 , + const unsigned i1 , + const ALL & , + const ALL & , + const ALL & , + const typename enable_if< ( + ViewAssignable< ViewTraits , ViewTraits >::assignable_value + && + is_same< typename ViewTraits::array_layout , LayoutRight >::value + && + ( ViewTraits::rank == 6 ) + && + ( ViewTraits::rank == 4 ) + && + ( ViewTraits::rank_dynamic == 4 ) + ), ALL >::type & ) + { + //typedef ViewTraits traits_type ; // unused + + dst.m_management.decrement( dst.m_ptr_on_device ); + + dst.m_management = src.m_management ; + dst.m_offset_map.N0 = src.m_offset_map.N2 ; + dst.m_offset_map.N1 = src.m_offset_map.N3 ; + dst.m_offset_map.N2 = src.m_offset_map.N4 ; + dst.m_offset_map.N3 = src.m_offset_map.N5 ; + dst.m_offset_map.SR = dst.m_offset_map.N1 * dst.m_offset_map.N2 * + dst.m_offset_map.N3 ; + dst.m_ptr_on_device = &src(i0,i1,0,0,0,0); + + dst.m_management.increment( dst.m_ptr_on_device ); + } + + //------------------------------------ + /** \brief Extract Rank-4 array from LayoutRight Rank-7 array. */ + template< class DT , class DL , class DD , class DM , + class ST , class SL , class SD , class SM > + KOKKOS_INLINE_FUNCTION + ViewAssignment( View & dst , + const View & src , + const unsigned i0 , + const unsigned i1 , + const unsigned i2 , + const ALL & , + const ALL & , + const ALL & , + const typename enable_if< ( + ViewAssignable< ViewTraits , ViewTraits >::assignable_value + && + is_same< typename ViewTraits::array_layout , LayoutRight >::value + && + ( ViewTraits::rank == 7 ) + && + ( ViewTraits::rank == 4 ) + && + ( ViewTraits::rank_dynamic == 4 ) + ), ALL >::type & ) + { + //typedef ViewTraits traits_type ; // unused + + dst.m_management.decrement( dst.m_ptr_on_device ); + + dst.m_management = src.m_management ; + dst.m_offset_map.N0 = src.m_offset_map.N3 ; + dst.m_offset_map.N1 = src.m_offset_map.N4 ; + dst.m_offset_map.N2 = src.m_offset_map.N5 ; + dst.m_offset_map.N3 = src.m_offset_map.N6 ; + dst.m_offset_map.SR = dst.m_offset_map.N1 * dst.m_offset_map.N2 * + dst.m_offset_map.N3 ; + dst.m_ptr_on_device = &src(i0,i1,i2,0,0,0,0); + + dst.m_management.increment( dst.m_ptr_on_device ); + } + + //------------------------------------ + /** \brief Extract Rank-4 array from LayoutRight Rank-8 array. */ + template< class DT , class DL , class DD , class DM , + class ST , class SL , class SD , class SM > + KOKKOS_INLINE_FUNCTION + ViewAssignment( View & dst , + const View & src , + const unsigned i0 , + const unsigned i1 , + const unsigned i2 , + const unsigned i3 , + const ALL & , + const ALL & , + const ALL & , + const typename enable_if< ( + ViewAssignable< ViewTraits , ViewTraits >::assignable_value + && + is_same< typename ViewTraits::array_layout , LayoutRight >::value + && + ( ViewTraits::rank == 8 ) + && + ( ViewTraits::rank == 4 ) + && + ( ViewTraits::rank_dynamic == 4 ) + ), ALL >::type & ) + { + //typedef ViewTraits traits_type ; // unused + + dst.m_management.decrement( dst.m_ptr_on_device ); + + dst.m_management = src.m_management ; + dst.m_offset_map.N0 = src.m_offset_map.N4 ; + dst.m_offset_map.N1 = src.m_offset_map.N5 ; + dst.m_offset_map.N2 = src.m_offset_map.N6 ; + dst.m_offset_map.N3 = src.m_offset_map.N7 ; + dst.m_offset_map.SR = dst.m_offset_map.N1 * dst.m_offset_map.N2 * + dst.m_offset_map.N3 ; + dst.m_ptr_on_device = &src(i0,i1,i2,i3,0,0,0,0); + + dst.m_management.increment( dst.m_ptr_on_device ); + } + + //------------------------------------ + /** \brief Extract Rank-5 array from LayoutRight Rank-6 array. */ + template< class DT , class DL , class DD , class DM , + class ST , class SL , class SD , class SM > + KOKKOS_INLINE_FUNCTION + ViewAssignment( View & dst , + const View & src , + const unsigned i0 , + const ALL & , + const ALL & , + const ALL & , + const ALL & , + const typename enable_if< ( + ViewAssignable< ViewTraits , ViewTraits >::assignable_value + && + is_same< typename ViewTraits::array_layout , LayoutRight >::value + && + ( ViewTraits::rank == 6 ) + && + ( ViewTraits::rank == 5 ) + && + ( ViewTraits::rank_dynamic == 5 ) + ), ALL >::type & ) + { + //typedef ViewTraits traits_type ; // unused + + dst.m_management.decrement( dst.m_ptr_on_device ); + + dst.m_management = src.m_management ; + dst.m_offset_map.N0 = src.m_offset_map.N1 ; + dst.m_offset_map.N1 = src.m_offset_map.N2 ; + dst.m_offset_map.N2 = src.m_offset_map.N3 ; + dst.m_offset_map.N3 = src.m_offset_map.N4 ; + dst.m_offset_map.N4 = src.m_offset_map.N5 ; + dst.m_offset_map.SR = dst.m_offset_map.N1 * dst.m_offset_map.N2 * + dst.m_offset_map.N3 * dst.m_offset_map.N4 ; + dst.m_ptr_on_device = &src(i0,0,0,0,0,0); + + dst.m_management.increment( dst.m_ptr_on_device ); + } + + //------------------------------------ + /** \brief Extract Rank-5 array from LayoutRight Rank-7 array. */ + template< class DT , class DL , class DD , class DM , + class ST , class SL , class SD , class SM > + KOKKOS_INLINE_FUNCTION + ViewAssignment( View & dst , + const View & src , + const unsigned i0 , + const unsigned i1 , + const ALL & , + const ALL & , + const ALL & , + const ALL & , + const typename enable_if< ( + ViewAssignable< ViewTraits , ViewTraits >::assignable_value + && + is_same< typename ViewTraits::array_layout , LayoutRight >::value + && + ( ViewTraits::rank == 7 ) + && + ( ViewTraits::rank == 5 ) + && + ( ViewTraits::rank_dynamic == 5 ) + ), ALL >::type & ) + { + //typedef ViewTraits traits_type ; // unused + + dst.m_management.decrement( dst.m_ptr_on_device ); + + dst.m_management = src.m_management ; + dst.m_offset_map.N0 = src.m_offset_map.N2 ; + dst.m_offset_map.N1 = src.m_offset_map.N3 ; + dst.m_offset_map.N2 = src.m_offset_map.N4 ; + dst.m_offset_map.N3 = src.m_offset_map.N5 ; + dst.m_offset_map.N4 = src.m_offset_map.N6 ; + dst.m_offset_map.SR = dst.m_offset_map.N1 * dst.m_offset_map.N2 * + dst.m_offset_map.N3 * dst.m_offset_map.N4 ; + dst.m_ptr_on_device = &src(i0,i1,0,0,0,0,0); + + dst.m_management.increment( dst.m_ptr_on_device ); + } + + //------------------------------------ + /** \brief Extract Rank-5 array from LayoutRight Rank-8 array. */ + template< class DT , class DL , class DD , class DM , + class ST , class SL , class SD , class SM > + KOKKOS_INLINE_FUNCTION + ViewAssignment( View & dst , + const View & src , + const unsigned i0 , + const unsigned i1 , + const unsigned i2 , + const ALL & , + const ALL & , + const ALL & , + const ALL & , + const typename enable_if< ( + ViewAssignable< ViewTraits , ViewTraits >::assignable_value + && + is_same< typename ViewTraits::array_layout , LayoutRight >::value + && + ( ViewTraits::rank == 8 ) + && + ( ViewTraits::rank == 5 ) + && + ( ViewTraits::rank_dynamic == 5 ) + ), ALL >::type & ) + { + //typedef ViewTraits traits_type ; // unused + + dst.m_management.decrement( dst.m_ptr_on_device ); + + dst.m_management = src.m_management ; + dst.m_offset_map.N0 = src.m_offset_map.N3 ; + dst.m_offset_map.N1 = src.m_offset_map.N4 ; + dst.m_offset_map.N2 = src.m_offset_map.N5 ; + dst.m_offset_map.N3 = src.m_offset_map.N6 ; + dst.m_offset_map.N4 = src.m_offset_map.N7 ; + dst.m_offset_map.SR = dst.m_offset_map.N1 * dst.m_offset_map.N2 * + dst.m_offset_map.N3 * dst.m_offset_map.N4 ; + dst.m_ptr_on_device = &src(i0,i1,i2,0,0,0,0,0); + + dst.m_management.increment( dst.m_ptr_on_device ); + } + + //------------------------------------ + + template< class DT , class DL , class DD , class DM + , class ST , class SL , class SD , class SM + , class Type0 + > + KOKKOS_INLINE_FUNCTION + ViewAssignment( View & dst , + const View & src , + const Type0 & arg0 , + const typename enable_if< ( + ViewAssignable< ViewTraits , ViewTraits >::assignable_value + && + is_same< typename ViewTraits::array_layout , LayoutStride >::value + && + ( ViewTraits::rank == 1 ) + && + ( unsigned(ViewTraits::rank) == + ( ViewOffsetRange< Type0 >::is_range ? 1u : 0 ) ) + )>::type * = 0 ) + { + enum { src_rank = 1 }; + + size_t str[2] = {0,0}; + + src.m_offset_map.stride( str ); + + const size_t offset = ViewOffsetRange< Type0 >::begin( arg0 ) * str[0] ; + + LayoutStride spec ; + + // Collapse dimension for non-ranges + if ( ViewOffsetRange< Type0 >::is_range ) { + spec.dimension[0] = ViewOffsetRange< Type0 >::dimension( src.m_offset_map.N0 , arg0 ); + spec.stride[0] = str[0] ; + } + else { + spec.dimension[0] = 1 ; + spec.stride[0] = 1 ; + } + + dst.m_management.decrement( dst.m_ptr_on_device ); + dst.m_management = src.m_management ; + dst.m_offset_map.assign( spec ); + dst.m_ptr_on_device = src.ptr_on_device() + offset ; + dst.m_management.increment( dst.m_ptr_on_device ); + } + + template< class DT , class DL , class DD , class DM + , class ST , class SL , class SD , class SM + , class Type0 + , class Type1 + > + KOKKOS_INLINE_FUNCTION + ViewAssignment( View & dst , + const View & src , + const Type0 & arg0 , + const Type1 & arg1 , + const typename enable_if< ( + ViewAssignable< ViewTraits , ViewTraits >::assignable_value + && + is_same< typename ViewTraits::array_layout , LayoutStride >::value + && + ( ViewTraits::rank == 2 ) + && + ( unsigned(ViewTraits::rank) == + ( ViewOffsetRange< Type0 >::is_range ? 1u : 0 ) + + ( ViewOffsetRange< Type1 >::is_range ? 1u : 0 ) ) + )>::type * = 0 ) + { + enum { src_rank = 2 }; + + const bool is_range[ src_rank ] = + { ViewOffsetRange< Type0 >::is_range + , ViewOffsetRange< Type1 >::is_range + }; + + const unsigned begin[ src_rank ] = + { static_cast(ViewOffsetRange< Type0 >::begin( arg0 )) + , static_cast(ViewOffsetRange< Type1 >::begin( arg1 )) + }; + + size_t stride[9] ; + + src.m_offset_map.stride( stride ); + + LayoutStride spec ; + + spec.dimension[0] = ViewOffsetRange< Type0 >::dimension( src.m_offset_map.N0 , arg0 ); + spec.dimension[1] = ViewOffsetRange< Type1 >::dimension( src.m_offset_map.N1 , arg1 ); + spec.stride[0] = stride[0] ; + spec.stride[1] = stride[1] ; + + size_t offset = 0 ; + + // Collapse dimension for non-ranges + for ( int i = 0 , j = 0 ; i < int(src_rank) ; ++i ) { + spec.dimension[j] = spec.dimension[i] ; + spec.stride[j] = spec.stride[i] ; + offset += begin[i] * spec.stride[i] ; + if ( is_range[i] ) { ++j ; } + } + + dst.m_management.decrement( dst.m_ptr_on_device ); + dst.m_management = src.m_management ; + dst.m_offset_map.assign( spec ); + dst.m_ptr_on_device = src.ptr_on_device() + offset ; + dst.m_management.increment( dst.m_ptr_on_device ); + } + + template< class DT , class DL , class DD , class DM + , class ST , class SL , class SD , class SM + , class Type0 + , class Type1 + , class Type2 + > + KOKKOS_INLINE_FUNCTION + ViewAssignment( View & dst , + const View & src , + const Type0 & arg0 , + const Type1 & arg1 , + const Type2 & arg2 , + const typename enable_if< ( + ViewAssignable< ViewTraits , ViewTraits >::assignable_value + && + is_same< typename ViewTraits::array_layout , LayoutStride >::value + && + ( ViewTraits::rank == 3 ) + && + ( unsigned(ViewTraits::rank) == + ( ViewOffsetRange< Type0 >::is_range ? 1u : 0 ) + + ( ViewOffsetRange< Type1 >::is_range ? 1u : 0 ) + + ( ViewOffsetRange< Type2 >::is_range ? 1u : 0 ) ) + )>::type * = 0 ) + { + enum { src_rank = 3 }; + + const bool is_range[ src_rank ] = + { ViewOffsetRange< Type0 >::is_range + , ViewOffsetRange< Type1 >::is_range + , ViewOffsetRange< Type2 >::is_range + }; + + // FIXME (mfh 26 Oct 2014) Should use size_type typedef here + // instead of unsigned. If we did that, the static_casts would be + // unnecessary. + const unsigned begin[ src_rank ] = { + static_cast (ViewOffsetRange< Type0 >::begin (arg0)) + , static_cast (ViewOffsetRange< Type1 >::begin (arg1)) + , static_cast (ViewOffsetRange< Type2 >::begin (arg2)) + }; + + // FIXME (mfh 26 Oct 2014) Should use size_type typedef here + // instead of unsigned. If we did that, the static_casts would be + // unnecessary. + unsigned dim[ src_rank ] = { + static_cast (ViewOffsetRange< Type0 >::dimension (src.m_offset_map.N0, arg0)) + , static_cast (ViewOffsetRange< Type1 >::dimension (src.m_offset_map.N1, arg1)) + , static_cast (ViewOffsetRange< Type2 >::dimension (src.m_offset_map.N2, arg2)) + }; + + size_t stride[9] = {0,0,0,0,0,0,0,0,0}; + + src.m_offset_map.stride( stride ); + + LayoutStride spec ; + + size_t offset = 0 ; + + // Collapse dimension for non-ranges + for ( int i = 0 , j = 0 ; i < int(src_rank) ; ++i ) { + spec.dimension[j] = dim[i] ; + spec.stride[j] = stride[i] ; + offset += begin[i] * stride[i] ; + if ( is_range[i] ) { ++j ; } + } + + dst.m_management.decrement( dst.m_ptr_on_device ); + dst.m_management = src.m_management ; + dst.m_offset_map.assign( spec ); + dst.m_ptr_on_device = src.ptr_on_device() + offset ; + dst.m_management.increment( dst.m_ptr_on_device ); + } + + template< class DT , class DL , class DD , class DM + , class ST , class SL , class SD , class SM + , class Type0 + , class Type1 + , class Type2 + , class Type3 + > + KOKKOS_INLINE_FUNCTION + ViewAssignment( View & dst , + const View & src , + const Type0 & arg0 , + const Type1 & arg1 , + const Type2 & arg2 , + const Type3 & arg3 , + const typename enable_if< ( + ViewAssignable< ViewTraits , ViewTraits >::assignable_value + && + is_same< typename ViewTraits::array_layout , LayoutStride >::value + && + ( ViewTraits::rank == 4 ) + && + ( unsigned(ViewTraits::rank) == + ( ViewOffsetRange< Type0 >::is_range ? 1u : 0 ) + + ( ViewOffsetRange< Type1 >::is_range ? 1u : 0 ) + + ( ViewOffsetRange< Type2 >::is_range ? 1u : 0 ) + + ( ViewOffsetRange< Type3 >::is_range ? 1u : 0 ) ) + )>::type * = 0 ) + { + enum { src_rank = 4 }; + const bool is_range[ src_rank ] = + { ViewOffsetRange< Type0 >::is_range + , ViewOffsetRange< Type1 >::is_range + , ViewOffsetRange< Type2 >::is_range + , ViewOffsetRange< Type3 >::is_range + }; + + const unsigned begin[ src_rank ] = + { static_cast(ViewOffsetRange< Type0 >::begin( arg0 )) + , static_cast(ViewOffsetRange< Type1 >::begin( arg1 )) + , static_cast(ViewOffsetRange< Type2 >::begin( arg2 )) + , static_cast(ViewOffsetRange< Type3 >::begin( arg3 )) + }; + + unsigned dim[ src_rank ] = + { static_cast(ViewOffsetRange< Type0 >::dimension( src.m_offset_map.N0 , arg0 )) + , static_cast(ViewOffsetRange< Type1 >::dimension( src.m_offset_map.N1 , arg1 )) + , static_cast(ViewOffsetRange< Type2 >::dimension( src.m_offset_map.N2 , arg2 )) + , static_cast(ViewOffsetRange< Type3 >::dimension( src.m_offset_map.N3 , arg3 )) + }; + + size_t stride[9] = {0,0,0,0,0,0,0,0,0}; + + src.m_offset_map.stride( stride ); + + LayoutStride spec ; + + size_t offset = 0 ; + + // Collapse dimension for non-ranges + for ( int i = 0 , j = 0 ; i < int(src_rank) ; ++i ) { + spec.dimension[j] = dim[i] ; + spec.stride[j] = stride[i] ; + offset += begin[i] * stride[i] ; + if ( is_range[i] ) { ++j ; } + } + + dst.m_management.decrement( dst.m_ptr_on_device ); + dst.m_management = src.m_management ; + dst.m_offset_map.assign( spec ); + dst.m_ptr_on_device = src.ptr_on_device() + offset ; + dst.m_management.increment( dst.m_ptr_on_device ); + } + + template< class DT , class DL , class DD , class DM + , class ST , class SL , class SD , class SM + , class Type0 + , class Type1 + , class Type2 + , class Type3 + , class Type4 + > + KOKKOS_INLINE_FUNCTION + ViewAssignment( View & dst , + const View & src , + const Type0 & arg0 , + const Type1 & arg1 , + const Type2 & arg2 , + const Type3 & arg3 , + const Type4 & arg4 , + const typename enable_if< ( + ViewAssignable< ViewTraits , ViewTraits >::assignable_value + && + is_same< typename ViewTraits::array_layout , LayoutStride >::value + && + ( ViewTraits::rank == 5 ) + && + ( unsigned(ViewTraits::rank) == + ( ViewOffsetRange< Type0 >::is_range ? 1u : 0 ) + + ( ViewOffsetRange< Type1 >::is_range ? 1u : 0 ) + + ( ViewOffsetRange< Type2 >::is_range ? 1u : 0 ) + + ( ViewOffsetRange< Type3 >::is_range ? 1u : 0 ) + + ( ViewOffsetRange< Type4 >::is_range ? 1u : 0 ) ) + )>::type * = 0 ) + { + enum { src_rank = 5 }; + const bool is_range[ src_rank ] = + { ViewOffsetRange< Type0 >::is_range + , ViewOffsetRange< Type1 >::is_range + , ViewOffsetRange< Type2 >::is_range + , ViewOffsetRange< Type3 >::is_range + , ViewOffsetRange< Type4 >::is_range + }; + + const unsigned begin[ src_rank ] = + { ViewOffsetRange< Type0 >::begin( arg0 ) + , ViewOffsetRange< Type1 >::begin( arg1 ) + , ViewOffsetRange< Type2 >::begin( arg2 ) + , ViewOffsetRange< Type3 >::begin( arg3 ) + , ViewOffsetRange< Type4 >::begin( arg4 ) + }; + + unsigned dim[ src_rank ] = + { ViewOffsetRange< Type0 >::dimension( src.m_offset_map.N0 , arg0 ) + , ViewOffsetRange< Type1 >::dimension( src.m_offset_map.N1 , arg1 ) + , ViewOffsetRange< Type2 >::dimension( src.m_offset_map.N2 , arg2 ) + , ViewOffsetRange< Type3 >::dimension( src.m_offset_map.N3 , arg3 ) + , ViewOffsetRange< Type4 >::dimension( src.m_offset_map.N4 , arg4 ) + }; + + size_t stride[9] = {0,0,0,0,0,0,0,0,0}; + + src.m_offset_map.stride( stride ); + + LayoutStride spec ; + + size_t offset = 0 ; + + // Collapse dimension for non-ranges + for ( int i = 0 , j = 0 ; i < int(src_rank) ; ++i ) { + spec.dimension[j] = dim[i] ; + spec.stride[j] = stride[i] ; + offset += begin[i] * stride[i] ; + if ( is_range[i] ) { ++j ; } + } + + dst.m_management.decrement( dst.m_ptr_on_device ); + dst.m_management = src.m_management ; + dst.m_offset_map.assign( spec ); + dst.m_ptr_on_device = src.ptr_on_device() + offset ; + dst.m_management.increment( dst.m_ptr_on_device ); + } + + template< class DT , class DL , class DD , class DM + , class ST , class SL , class SD , class SM + , class Type0 + , class Type1 + , class Type2 + , class Type3 + , class Type4 + , class Type5 + > + KOKKOS_INLINE_FUNCTION + ViewAssignment( View & dst , + const View & src , + const Type0 & arg0 , + const Type1 & arg1 , + const Type2 & arg2 , + const Type3 & arg3 , + const Type4 & arg4 , + const Type5 & arg5 , + const typename enable_if< ( + ViewAssignable< ViewTraits , ViewTraits >::assignable_value + && + is_same< typename ViewTraits::array_layout , LayoutStride >::value + && + ( ViewTraits::rank == 6 ) + && + ( unsigned(ViewTraits::rank) == + ( ViewOffsetRange< Type0 >::is_range ? 1u : 0 ) + + ( ViewOffsetRange< Type1 >::is_range ? 1u : 0 ) + + ( ViewOffsetRange< Type2 >::is_range ? 1u : 0 ) + + ( ViewOffsetRange< Type3 >::is_range ? 1u : 0 ) + + ( ViewOffsetRange< Type4 >::is_range ? 1u : 0 ) + + ( ViewOffsetRange< Type5 >::is_range ? 1u : 0 ) ) + )>::type * = 0 ) + { + enum { src_rank = 6 }; + const bool is_range[ src_rank ] = + { ViewOffsetRange< Type0 >::is_range + , ViewOffsetRange< Type1 >::is_range + , ViewOffsetRange< Type2 >::is_range + , ViewOffsetRange< Type3 >::is_range + , ViewOffsetRange< Type4 >::is_range + , ViewOffsetRange< Type5 >::is_range + }; + + const unsigned begin[ src_rank ] = + { ViewOffsetRange< Type0 >::begin( arg0 ) + , ViewOffsetRange< Type1 >::begin( arg1 ) + , ViewOffsetRange< Type2 >::begin( arg2 ) + , ViewOffsetRange< Type3 >::begin( arg3 ) + , ViewOffsetRange< Type4 >::begin( arg4 ) + , ViewOffsetRange< Type5 >::begin( arg5 ) + }; + + unsigned dim[ src_rank ] = + { ViewOffsetRange< Type0 >::dimension( src.m_offset_map.N0 , arg0 ) + , ViewOffsetRange< Type1 >::dimension( src.m_offset_map.N1 , arg1 ) + , ViewOffsetRange< Type2 >::dimension( src.m_offset_map.N2 , arg2 ) + , ViewOffsetRange< Type3 >::dimension( src.m_offset_map.N3 , arg3 ) + , ViewOffsetRange< Type4 >::dimension( src.m_offset_map.N4 , arg4 ) + , ViewOffsetRange< Type5 >::dimension( src.m_offset_map.N5 , arg5 ) + }; + + size_t stride[9] = {0,0,0,0,0,0,0,0,0}; + + src.m_offset_map.stride( stride ); + + LayoutStride spec ; + + size_t offset = 0 ; + + // Collapse dimension for non-ranges + for ( int i = 0 , j = 0 ; i < int(src_rank) ; ++i ) { + spec.dimension[j] = dim[i] ; + spec.stride[j] = stride[i] ; + offset += begin[i] * stride[i] ; + if ( is_range[i] ) { ++j ; } + } + + dst.m_management.decrement( dst.m_ptr_on_device ); + dst.m_management = src.m_management ; + dst.m_offset_map.assign( spec ); + dst.m_ptr_on_device = src.ptr_on_device() + offset ; + dst.m_management.increment( dst.m_ptr_on_device ); + } + + template< class DT , class DL , class DD , class DM + , class ST , class SL , class SD , class SM + , class Type0 + , class Type1 + , class Type2 + , class Type3 + , class Type4 + , class Type5 + , class Type6 + > + KOKKOS_INLINE_FUNCTION + ViewAssignment( View & dst , + const View & src , + const Type0 & arg0 , + const Type1 & arg1 , + const Type2 & arg2 , + const Type3 & arg3 , + const Type4 & arg4 , + const Type5 & arg5 , + const Type6 & arg6 , + const typename enable_if< ( + ViewAssignable< ViewTraits , ViewTraits >::assignable_value + && + is_same< typename ViewTraits::array_layout , LayoutStride >::value + && + ( ViewTraits::rank == 7 ) + && + ( unsigned(ViewTraits::rank) == + ( ViewOffsetRange< Type0 >::is_range ? 1u : 0 ) + + ( ViewOffsetRange< Type1 >::is_range ? 1u : 0 ) + + ( ViewOffsetRange< Type2 >::is_range ? 1u : 0 ) + + ( ViewOffsetRange< Type3 >::is_range ? 1u : 0 ) + + ( ViewOffsetRange< Type4 >::is_range ? 1u : 0 ) + + ( ViewOffsetRange< Type5 >::is_range ? 1u : 0 ) + + ( ViewOffsetRange< Type6 >::is_range ? 1u : 0 ) ) + )>::type * = 0 ) + { + enum { src_rank = 7 }; + const bool is_range[ src_rank ] = + { ViewOffsetRange< Type0 >::is_range + , ViewOffsetRange< Type1 >::is_range + , ViewOffsetRange< Type2 >::is_range + , ViewOffsetRange< Type3 >::is_range + , ViewOffsetRange< Type4 >::is_range + , ViewOffsetRange< Type5 >::is_range + , ViewOffsetRange< Type6 >::is_range + }; + + const unsigned begin[ src_rank ] = + { ViewOffsetRange< Type0 >::begin( arg0 ) + , ViewOffsetRange< Type1 >::begin( arg1 ) + , ViewOffsetRange< Type2 >::begin( arg2 ) + , ViewOffsetRange< Type3 >::begin( arg3 ) + , ViewOffsetRange< Type4 >::begin( arg4 ) + , ViewOffsetRange< Type5 >::begin( arg5 ) + , ViewOffsetRange< Type6 >::begin( arg6 ) + }; + + unsigned dim[ src_rank ] = + { ViewOffsetRange< Type0 >::dimension( src.m_offset_map.N0 , arg0 ) + , ViewOffsetRange< Type1 >::dimension( src.m_offset_map.N1 , arg1 ) + , ViewOffsetRange< Type2 >::dimension( src.m_offset_map.N2 , arg2 ) + , ViewOffsetRange< Type3 >::dimension( src.m_offset_map.N3 , arg3 ) + , ViewOffsetRange< Type4 >::dimension( src.m_offset_map.N4 , arg4 ) + , ViewOffsetRange< Type5 >::dimension( src.m_offset_map.N5 , arg5 ) + , ViewOffsetRange< Type6 >::dimension( src.m_offset_map.N6 , arg6 ) + }; + + size_t stride[9] = {0,0,0,0,0,0,0,0,0}; + + src.m_offset_map.stride( stride ); + + LayoutStride spec ; + + size_t offset = 0 ; + + // Collapse dimension for non-ranges + for ( int i = 0 , j = 0 ; i < int(src_rank) ; ++i ) { + spec.dimension[j] = dim[i] ; + spec.stride[j] = stride[i] ; + offset += begin[i] * stride[i] ; + if ( is_range[i] ) { ++j ; } + } + + dst.m_management.decrement( dst.m_ptr_on_device ); + dst.m_management = src.m_management ; + dst.m_offset_map.assign( spec ); + dst.m_ptr_on_device = src.ptr_on_device() + offset ; + dst.m_management.increment( dst.m_ptr_on_device ); + } + + template< class DT , class DL , class DD , class DM + , class ST , class SL , class SD , class SM + , class Type0 + , class Type1 + , class Type2 + , class Type3 + , class Type4 + , class Type5 + , class Type6 + , class Type7 + > + KOKKOS_INLINE_FUNCTION + ViewAssignment( View & dst , + const View & src , + const Type0 & arg0 , + const Type1 & arg1 , + const Type2 & arg2 , + const Type3 & arg3 , + const Type4 & arg4 , + const Type5 & arg5 , + const Type6 & arg6 , + const Type7 & arg7 , + const typename enable_if< ( + ViewAssignable< ViewTraits , ViewTraits >::assignable_value + && + is_same< typename ViewTraits::array_layout , LayoutStride >::value + && + ( ViewTraits::rank == 8 ) + && + ( unsigned(ViewTraits::rank) == + ( ViewOffsetRange< Type0 >::is_range ? 1u : 0 ) + + ( ViewOffsetRange< Type1 >::is_range ? 1u : 0 ) + + ( ViewOffsetRange< Type2 >::is_range ? 1u : 0 ) + + ( ViewOffsetRange< Type3 >::is_range ? 1u : 0 ) + + ( ViewOffsetRange< Type4 >::is_range ? 1u : 0 ) + + ( ViewOffsetRange< Type5 >::is_range ? 1u : 0 ) + + ( ViewOffsetRange< Type6 >::is_range ? 1u : 0 ) + + ( ViewOffsetRange< Type7 >::is_range ? 1u : 0 ) ) + )>::type * = 0 ) + { + enum { src_rank = 8 }; + + const bool is_range[ src_rank ] = + { ViewOffsetRange< Type0 >::is_range + , ViewOffsetRange< Type1 >::is_range + , ViewOffsetRange< Type2 >::is_range + , ViewOffsetRange< Type3 >::is_range + , ViewOffsetRange< Type4 >::is_range + , ViewOffsetRange< Type5 >::is_range + , ViewOffsetRange< Type6 >::is_range + , ViewOffsetRange< Type7 >::is_range + }; + + const unsigned begin[ src_rank ] = + { ViewOffsetRange< Type0 >::begin( arg0 ) + , ViewOffsetRange< Type1 >::begin( arg1 ) + , ViewOffsetRange< Type2 >::begin( arg2 ) + , ViewOffsetRange< Type3 >::begin( arg3 ) + , ViewOffsetRange< Type4 >::begin( arg4 ) + , ViewOffsetRange< Type5 >::begin( arg5 ) + , ViewOffsetRange< Type6 >::begin( arg6 ) + , ViewOffsetRange< Type7 >::begin( arg7 ) + }; + + unsigned dim[ src_rank ] = + { ViewOffsetRange< Type0 >::dimension( src.m_offset_map.N0 , arg0 ) + , ViewOffsetRange< Type1 >::dimension( src.m_offset_map.N1 , arg1 ) + , ViewOffsetRange< Type2 >::dimension( src.m_offset_map.N2 , arg2 ) + , ViewOffsetRange< Type3 >::dimension( src.m_offset_map.N3 , arg3 ) + , ViewOffsetRange< Type4 >::dimension( src.m_offset_map.N4 , arg4 ) + , ViewOffsetRange< Type5 >::dimension( src.m_offset_map.N5 , arg5 ) + , ViewOffsetRange< Type6 >::dimension( src.m_offset_map.N6 , arg6 ) + , ViewOffsetRange< Type7 >::dimension( src.m_offset_map.N7 , arg7 ) + }; + + size_t stride[9] = {0,0,0,0,0,0,0,0,0}; + + src.m_offset_map.stride( stride ); + + LayoutStride spec ; + + size_t offset = 0 ; + + // Collapse dimension for non-ranges + for ( int i = 0 , j = 0 ; i < int(src_rank) ; ++i ) { + spec.dimension[j] = dim[i] ; + spec.stride[j] = stride[i] ; + offset += begin[i] * stride[i] ; + if ( is_range[i] ) { ++j ; } + } + + dst.m_management.decrement( dst.m_ptr_on_device ); + + dst.m_management = src.m_management ; + + dst.m_offset_map.assign( spec ); + + dst.m_ptr_on_device = src.ptr_on_device() + offset ; + + dst.m_management.increment( dst.m_ptr_on_device ); + } + + //------------------------------------ + /** \brief Deep copy data from compatible value type, layout, rank, and specialization. + * Check the dimensions and allocation lengths at runtime. + */ + template< class DT , class DL , class DD , class DM , + class ST , class SL , class SD , class SM > + inline static + void deep_copy( const View & dst , + const View & src , + const typename Impl::enable_if<( + Impl::is_same< typename ViewTraits::value_type , + typename ViewTraits::non_const_value_type >::value + && + Impl::is_same< typename ViewTraits::array_layout , + typename ViewTraits::array_layout >::value + && + ( unsigned(ViewTraits::rank) == unsigned(ViewTraits::rank) ) + )>::type * = 0 ) + { + typedef typename ViewTraits::memory_space dst_memory_space ; + typedef typename ViewTraits::memory_space src_memory_space ; + + if ( dst.ptr_on_device() != src.ptr_on_device() ) { + + Impl::assert_shapes_are_equal( dst.m_offset_map , src.m_offset_map ); + + const size_t nbytes = dst.m_offset_map.scalar_size * dst.m_offset_map.capacity(); + + DeepCopy< dst_memory_space , src_memory_space >( dst.ptr_on_device() , src.ptr_on_device() , nbytes ); + } + } +}; + +} /* namespace Impl */ +} /* namespace Kokkos */ + +//---------------------------------------------------------------------------- +//---------------------------------------------------------------------------- + +namespace Kokkos { +namespace Impl { + +template< class SrcDataType , class SrcArg1Type , class SrcArg2Type , class SrcArg3Type + , class SubArg0_type , class SubArg1_type , class SubArg2_type , class SubArg3_type + , class SubArg4_type , class SubArg5_type , class SubArg6_type , class SubArg7_type + > +struct ViewSubview< View< SrcDataType , SrcArg1Type , SrcArg2Type , SrcArg3Type , Impl::ViewDefault > + , SubArg0_type , SubArg1_type , SubArg2_type , SubArg3_type + , SubArg4_type , SubArg5_type , SubArg6_type , SubArg7_type > +{ +private: + + typedef View< SrcDataType , SrcArg1Type , SrcArg2Type , SrcArg3Type , Impl::ViewDefault > SrcViewType ; + + enum { V0 = Impl::is_same< SubArg0_type , void >::value ? 1 : 0 }; + enum { V1 = Impl::is_same< SubArg1_type , void >::value ? 1 : 0 }; + enum { V2 = Impl::is_same< SubArg2_type , void >::value ? 1 : 0 }; + enum { V3 = Impl::is_same< SubArg3_type , void >::value ? 1 : 0 }; + enum { V4 = Impl::is_same< SubArg4_type , void >::value ? 1 : 0 }; + enum { V5 = Impl::is_same< SubArg5_type , void >::value ? 1 : 0 }; + enum { V6 = Impl::is_same< SubArg6_type , void >::value ? 1 : 0 }; + enum { V7 = Impl::is_same< SubArg7_type , void >::value ? 1 : 0 }; + + // The source view rank must be equal to the input argument rank + // Once a void argument is encountered all subsequent arguments must be void. + enum { InputRank = + Impl::StaticAssert<( SrcViewType::rank == + ( V0 ? 0 : ( + V1 ? 1 : ( + V2 ? 2 : ( + V3 ? 3 : ( + V4 ? 4 : ( + V5 ? 5 : ( + V6 ? 6 : ( + V7 ? 7 : 8 ))))))) )) + && + ( SrcViewType::rank == + ( 8 - ( V0 + V1 + V2 + V3 + V4 + V5 + V6 + V7 ) ) ) + >::value ? SrcViewType::rank : 0 }; + + enum { R0 = Impl::ViewOffsetRange< SubArg0_type >::is_range ? 1 : 0 }; + enum { R1 = Impl::ViewOffsetRange< SubArg1_type >::is_range ? 1 : 0 }; + enum { R2 = Impl::ViewOffsetRange< SubArg2_type >::is_range ? 1 : 0 }; + enum { R3 = Impl::ViewOffsetRange< SubArg3_type >::is_range ? 1 : 0 }; + enum { R4 = Impl::ViewOffsetRange< SubArg4_type >::is_range ? 1 : 0 }; + enum { R5 = Impl::ViewOffsetRange< SubArg5_type >::is_range ? 1 : 0 }; + enum { R6 = Impl::ViewOffsetRange< SubArg6_type >::is_range ? 1 : 0 }; + enum { R7 = Impl::ViewOffsetRange< SubArg7_type >::is_range ? 1 : 0 }; + + enum { OutputRank = unsigned(R0) + unsigned(R1) + unsigned(R2) + unsigned(R3) + + unsigned(R4) + unsigned(R5) + unsigned(R6) + unsigned(R7) }; + + // Reverse + enum { R0_rev = 0 == InputRank ? 0u : ( + 1 == InputRank ? unsigned(R0) : ( + 2 == InputRank ? unsigned(R1) : ( + 3 == InputRank ? unsigned(R2) : ( + 4 == InputRank ? unsigned(R3) : ( + 5 == InputRank ? unsigned(R4) : ( + 6 == InputRank ? unsigned(R5) : ( + 7 == InputRank ? unsigned(R6) : unsigned(R7) ))))))) }; + + typedef typename SrcViewType::array_layout SrcViewLayout ; + + // Choose array layout, attempting to preserve original layout if at all possible. + typedef typename Impl::if_c< + ( // Same Layout IF + // OutputRank 0 + ( OutputRank == 0 ) + || + // OutputRank 1 or 2, InputLayout Left, Interval 0 + // because single stride one or second index has a stride. + ( OutputRank <= 2 && R0 && Impl::is_same::value ) + || + // OutputRank 1 or 2, InputLayout Right, Interval [InputRank-1] + // because single stride one or second index has a stride. + ( OutputRank <= 2 && R0_rev && Impl::is_same::value ) + ), SrcViewLayout , Kokkos::LayoutStride >::type OutputViewLayout ; + + // Choose data type as a purely dynamic rank array to accomodate a runtime range. + typedef typename Impl::if_c< OutputRank == 0 , typename SrcViewType::value_type , + typename Impl::if_c< OutputRank == 1 , typename SrcViewType::value_type *, + typename Impl::if_c< OutputRank == 2 , typename SrcViewType::value_type **, + typename Impl::if_c< OutputRank == 3 , typename SrcViewType::value_type ***, + typename Impl::if_c< OutputRank == 4 , typename SrcViewType::value_type ****, + typename Impl::if_c< OutputRank == 5 , typename SrcViewType::value_type *****, + typename Impl::if_c< OutputRank == 6 , typename SrcViewType::value_type ******, + typename Impl::if_c< OutputRank == 7 , typename SrcViewType::value_type *******, + typename SrcViewType::value_type ******** + >::type >::type >::type >::type >::type >::type >::type >::type OutputData ; + + // Choose space. + // If the source view's template arg1 or arg2 is a space then use it, + // otherwise use the source view's execution space. + + typedef typename Impl::if_c< Impl::is_space< SrcArg1Type >::value , SrcArg1Type , + typename Impl::if_c< Impl::is_space< SrcArg2Type >::value , SrcArg2Type , typename SrcViewType::execution_space + >::type >::type OutputSpace ; + +public: + + // If keeping the layout then match non-data type arguments + // else keep execution space and memory traits. + typedef typename + Impl::if_c< Impl::is_same< SrcViewLayout , OutputViewLayout >::value + , Kokkos::View< OutputData , SrcArg1Type , SrcArg2Type , SrcArg3Type , Impl::ViewDefault > + , Kokkos::View< OutputData , OutputViewLayout , OutputSpace + , typename SrcViewType::memory_traits + , Impl::ViewDefault > + >::type type ; +}; + +} /* namespace Impl */ +} /* namespace Kokkos */ + +//---------------------------------------------------------------------------- +//---------------------------------------------------------------------------- + +namespace Kokkos { + +// Construct subview of a Rank 8 view +template< class DstDataType , class DstArg1Type , class DstArg2Type , class DstArg3Type > +template< class SrcDataType , class SrcArg1Type , class SrcArg2Type , class SrcArg3Type + , class SubArg0_type , class SubArg1_type , class SubArg2_type , class SubArg3_type + , class SubArg4_type , class SubArg5_type , class SubArg6_type , class SubArg7_type + > +KOKKOS_INLINE_FUNCTION +View< DstDataType , DstArg1Type , DstArg2Type , DstArg3Type , Impl::ViewDefault >:: +View( const View< SrcDataType , SrcArg1Type , SrcArg2Type , SrcArg3Type , Impl::ViewDefault > & src + , const SubArg0_type & arg0 + , const SubArg1_type & arg1 + , const SubArg2_type & arg2 + , const SubArg3_type & arg3 + , const SubArg4_type & arg4 + , const SubArg5_type & arg5 + , const SubArg6_type & arg6 + , const SubArg7_type & arg7 + ) + : m_ptr_on_device( (typename traits::value_type*) NULL) + , m_offset_map() + , m_management() +{ + // This constructor can only be used to construct a subview + // from the source view. This type must match the subview type + // deduced from the source view and subview arguments. + + typedef Impl::ViewSubview< View< SrcDataType , SrcArg1Type , SrcArg2Type , SrcArg3Type , Impl::ViewDefault > + , SubArg0_type , SubArg1_type , SubArg2_type , SubArg3_type + , SubArg4_type , SubArg5_type , SubArg6_type , SubArg7_type > + ViewSubviewDeduction ; + + enum { is_a_valid_subview_constructor = + Impl::StaticAssert< + Impl::is_same< View , typename ViewSubviewDeduction::type >::value + >::value + }; + + if ( is_a_valid_subview_constructor ) { + + typedef Impl::ViewOffsetRange< SubArg0_type > R0 ; + typedef Impl::ViewOffsetRange< SubArg1_type > R1 ; + typedef Impl::ViewOffsetRange< SubArg2_type > R2 ; + typedef Impl::ViewOffsetRange< SubArg3_type > R3 ; + typedef Impl::ViewOffsetRange< SubArg4_type > R4 ; + typedef Impl::ViewOffsetRange< SubArg5_type > R5 ; + typedef Impl::ViewOffsetRange< SubArg6_type > R6 ; + typedef Impl::ViewOffsetRange< SubArg7_type > R7 ; + + // 'assign_subview' returns whether the subview offset_map + // introduces noncontiguity in the view. + const bool introduce_noncontiguity = + m_offset_map.assign_subview( src.m_offset_map + , R0::dimension( src.m_offset_map.N0 , arg0 ) + , R1::dimension( src.m_offset_map.N1 , arg1 ) + , R2::dimension( src.m_offset_map.N2 , arg2 ) + , R3::dimension( src.m_offset_map.N3 , arg3 ) + , R4::dimension( src.m_offset_map.N4 , arg4 ) + , R5::dimension( src.m_offset_map.N5 , arg5 ) + , R6::dimension( src.m_offset_map.N6 , arg6 ) + , R7::dimension( src.m_offset_map.N7 , arg7 ) + ); + + if ( m_offset_map.capacity() ) { + + m_management = src.m_management ; + + if ( introduce_noncontiguity ) m_management.set_noncontiguous(); + + m_ptr_on_device = src.m_ptr_on_device + + src.m_offset_map( R0::begin( arg0 ) + , R1::begin( arg1 ) + , R2::begin( arg2 ) + , R3::begin( arg3 ) + , R4::begin( arg4 ) + , R5::begin( arg5 ) + , R6::begin( arg6 ) + , R7::begin( arg7 ) ); + m_management.increment( m_ptr_on_device ); + } + } +} + +// Construct subview of a Rank 7 view +template< class DstDataType , class DstArg1Type , class DstArg2Type , class DstArg3Type > +template< class SrcDataType , class SrcArg1Type , class SrcArg2Type , class SrcArg3Type + , class SubArg0_type , class SubArg1_type , class SubArg2_type , class SubArg3_type + , class SubArg4_type , class SubArg5_type , class SubArg6_type + > +KOKKOS_INLINE_FUNCTION +View< DstDataType , DstArg1Type , DstArg2Type , DstArg3Type , Impl::ViewDefault >:: +View( const View< SrcDataType , SrcArg1Type , SrcArg2Type , SrcArg3Type , Impl::ViewDefault > & src + , const SubArg0_type & arg0 + , const SubArg1_type & arg1 + , const SubArg2_type & arg2 + , const SubArg3_type & arg3 + , const SubArg4_type & arg4 + , const SubArg5_type & arg5 + , const SubArg6_type & arg6 + ) + : m_ptr_on_device( (typename traits::value_type*) NULL) + , m_offset_map() + , m_management() +{ + // This constructor can only be used to construct a subview + // from the source view. This type must match the subview type + // deduced from the source view and subview arguments. + + typedef Impl::ViewSubview< View< SrcDataType , SrcArg1Type , SrcArg2Type , SrcArg3Type , Impl::ViewDefault > + , SubArg0_type , SubArg1_type , SubArg2_type , SubArg3_type + , SubArg4_type , SubArg5_type , SubArg6_type , void > + ViewSubviewDeduction ; + + enum { is_a_valid_subview_constructor = + Impl::StaticAssert< + Impl::is_same< View , typename ViewSubviewDeduction::type >::value + >::value + }; + + if ( is_a_valid_subview_constructor ) { + + typedef Impl::ViewOffsetRange< SubArg0_type > R0 ; + typedef Impl::ViewOffsetRange< SubArg1_type > R1 ; + typedef Impl::ViewOffsetRange< SubArg2_type > R2 ; + typedef Impl::ViewOffsetRange< SubArg3_type > R3 ; + typedef Impl::ViewOffsetRange< SubArg4_type > R4 ; + typedef Impl::ViewOffsetRange< SubArg5_type > R5 ; + typedef Impl::ViewOffsetRange< SubArg6_type > R6 ; + + // 'assign_subview' returns whether the subview offset_map + // introduces noncontiguity in the view. + const bool introduce_noncontiguity = + m_offset_map.assign_subview( src.m_offset_map + , R0::dimension( src.m_offset_map.N0 , arg0 ) + , R1::dimension( src.m_offset_map.N1 , arg1 ) + , R2::dimension( src.m_offset_map.N2 , arg2 ) + , R3::dimension( src.m_offset_map.N3 , arg3 ) + , R4::dimension( src.m_offset_map.N4 , arg4 ) + , R5::dimension( src.m_offset_map.N5 , arg5 ) + , R6::dimension( src.m_offset_map.N6 , arg6 ) + , 0 + ); + + if ( m_offset_map.capacity() ) { + + m_management = src.m_management ; + + if ( introduce_noncontiguity ) m_management.set_noncontiguous(); + + m_ptr_on_device = src.m_ptr_on_device + + src.m_offset_map( R0::begin( arg0 ) + , R1::begin( arg1 ) + , R2::begin( arg2 ) + , R3::begin( arg3 ) + , R4::begin( arg4 ) + , R5::begin( arg5 ) + , R6::begin( arg6 ) + , 0 ); + m_management.increment( m_ptr_on_device ); + } + } +} + +// Construct subview of a Rank 6 view +template< class DstDataType , class DstArg1Type , class DstArg2Type , class DstArg3Type > +template< class SrcDataType , class SrcArg1Type , class SrcArg2Type , class SrcArg3Type + , class SubArg0_type , class SubArg1_type , class SubArg2_type , class SubArg3_type + , class SubArg4_type , class SubArg5_type + > +KOKKOS_INLINE_FUNCTION +View< DstDataType , DstArg1Type , DstArg2Type , DstArg3Type , Impl::ViewDefault >:: +View( const View< SrcDataType , SrcArg1Type , SrcArg2Type , SrcArg3Type , Impl::ViewDefault > & src + , const SubArg0_type & arg0 + , const SubArg1_type & arg1 + , const SubArg2_type & arg2 + , const SubArg3_type & arg3 + , const SubArg4_type & arg4 + , const SubArg5_type & arg5 + ) + : m_ptr_on_device( (typename traits::value_type*) NULL) + , m_offset_map() + , m_management() +{ + // This constructor can only be used to construct a subview + // from the source view. This type must match the subview type + // deduced from the source view and subview arguments. + + typedef Impl::ViewSubview< View< SrcDataType , SrcArg1Type , SrcArg2Type , SrcArg3Type , Impl::ViewDefault > + , SubArg0_type , SubArg1_type , SubArg2_type , SubArg3_type + , SubArg4_type , SubArg5_type , void , void > + ViewSubviewDeduction ; + + enum { is_a_valid_subview_constructor = + Impl::StaticAssert< + Impl::is_same< View , typename ViewSubviewDeduction::type >::value + >::value + }; + + if ( is_a_valid_subview_constructor ) { + + typedef Impl::ViewOffsetRange< SubArg0_type > R0 ; + typedef Impl::ViewOffsetRange< SubArg1_type > R1 ; + typedef Impl::ViewOffsetRange< SubArg2_type > R2 ; + typedef Impl::ViewOffsetRange< SubArg3_type > R3 ; + typedef Impl::ViewOffsetRange< SubArg4_type > R4 ; + typedef Impl::ViewOffsetRange< SubArg5_type > R5 ; + + // 'assign_subview' returns whether the subview offset_map + // introduces noncontiguity in the view. + const bool introduce_noncontiguity = + m_offset_map.assign_subview( src.m_offset_map + , R0::dimension( src.m_offset_map.N0 , arg0 ) + , R1::dimension( src.m_offset_map.N1 , arg1 ) + , R2::dimension( src.m_offset_map.N2 , arg2 ) + , R3::dimension( src.m_offset_map.N3 , arg3 ) + , R4::dimension( src.m_offset_map.N4 , arg4 ) + , R5::dimension( src.m_offset_map.N5 , arg5 ) + , 0 + , 0 + ); + + if ( m_offset_map.capacity() ) { + + m_management = src.m_management ; + + if ( introduce_noncontiguity ) m_management.set_noncontiguous(); + + m_ptr_on_device = src.m_ptr_on_device + + src.m_offset_map( R0::begin( arg0 ) + , R1::begin( arg1 ) + , R2::begin( arg2 ) + , R3::begin( arg3 ) + , R4::begin( arg4 ) + , R5::begin( arg5 ) + , 0 + , 0 ); + m_management.increment( m_ptr_on_device ); + } + } +} + +// Construct subview of a Rank 5 view +template< class DstDataType , class DstArg1Type , class DstArg2Type , class DstArg3Type > +template< class SrcDataType , class SrcArg1Type , class SrcArg2Type , class SrcArg3Type + , class SubArg0_type , class SubArg1_type , class SubArg2_type , class SubArg3_type + , class SubArg4_type + > +KOKKOS_INLINE_FUNCTION +View< DstDataType , DstArg1Type , DstArg2Type , DstArg3Type , Impl::ViewDefault >:: +View( const View< SrcDataType , SrcArg1Type , SrcArg2Type , SrcArg3Type , Impl::ViewDefault > & src + , const SubArg0_type & arg0 + , const SubArg1_type & arg1 + , const SubArg2_type & arg2 + , const SubArg3_type & arg3 + , const SubArg4_type & arg4 + ) + : m_ptr_on_device( (typename traits::value_type*) NULL) + , m_offset_map() + , m_management() +{ + // This constructor can only be used to construct a subview + // from the source view. This type must match the subview type + // deduced from the source view and subview arguments. + + typedef Impl::ViewSubview< View< SrcDataType , SrcArg1Type , SrcArg2Type , SrcArg3Type , Impl::ViewDefault > + , SubArg0_type , SubArg1_type , SubArg2_type , SubArg3_type + , SubArg4_type , void , void , void > + ViewSubviewDeduction ; + + enum { is_a_valid_subview_constructor = + Impl::StaticAssert< + Impl::is_same< View , typename ViewSubviewDeduction::type >::value + >::value + }; + + if ( is_a_valid_subview_constructor ) { + + typedef Impl::ViewOffsetRange< SubArg0_type > R0 ; + typedef Impl::ViewOffsetRange< SubArg1_type > R1 ; + typedef Impl::ViewOffsetRange< SubArg2_type > R2 ; + typedef Impl::ViewOffsetRange< SubArg3_type > R3 ; + typedef Impl::ViewOffsetRange< SubArg4_type > R4 ; + + // 'assign_subview' returns whether the subview offset_map + // introduces noncontiguity in the view. + const bool introduce_noncontiguity = + m_offset_map.assign_subview( src.m_offset_map + , R0::dimension( src.m_offset_map.N0 , arg0 ) + , R1::dimension( src.m_offset_map.N1 , arg1 ) + , R2::dimension( src.m_offset_map.N2 , arg2 ) + , R3::dimension( src.m_offset_map.N3 , arg3 ) + , R4::dimension( src.m_offset_map.N4 , arg4 ) + , 0 + , 0 + , 0 + ); + + if ( m_offset_map.capacity() ) { + + m_management = src.m_management ; + + if ( introduce_noncontiguity ) m_management.set_noncontiguous(); + + m_ptr_on_device = src.m_ptr_on_device + + src.m_offset_map( R0::begin( arg0 ) + , R1::begin( arg1 ) + , R2::begin( arg2 ) + , R3::begin( arg3 ) + , R4::begin( arg4 ) + , 0 + , 0 + , 0 ); + m_management.increment( m_ptr_on_device ); + } + } +} + +// Construct subview of a Rank 4 view +template< class DstDataType , class DstArg1Type , class DstArg2Type , class DstArg3Type > +template< class SrcDataType , class SrcArg1Type , class SrcArg2Type , class SrcArg3Type + , class SubArg0_type , class SubArg1_type , class SubArg2_type , class SubArg3_type + > +KOKKOS_INLINE_FUNCTION +View< DstDataType , DstArg1Type , DstArg2Type , DstArg3Type , Impl::ViewDefault >:: +View( const View< SrcDataType , SrcArg1Type , SrcArg2Type , SrcArg3Type , Impl::ViewDefault > & src + , const SubArg0_type & arg0 + , const SubArg1_type & arg1 + , const SubArg2_type & arg2 + , const SubArg3_type & arg3 + ) + : m_ptr_on_device( (typename traits::value_type*) NULL) + , m_offset_map() + , m_management() +{ + // This constructor can only be used to construct a subview + // from the source view. This type must match the subview type + // deduced from the source view and subview arguments. + + typedef Impl::ViewSubview< View< SrcDataType , SrcArg1Type , SrcArg2Type , SrcArg3Type , Impl::ViewDefault > + , SubArg0_type , SubArg1_type , SubArg2_type , SubArg3_type + , void , void , void , void > + ViewSubviewDeduction ; + + enum { is_a_valid_subview_constructor = + Impl::StaticAssert< + Impl::is_same< View , typename ViewSubviewDeduction::type >::value + >::value + }; + + if ( is_a_valid_subview_constructor ) { + + typedef Impl::ViewOffsetRange< SubArg0_type > R0 ; + typedef Impl::ViewOffsetRange< SubArg1_type > R1 ; + typedef Impl::ViewOffsetRange< SubArg2_type > R2 ; + typedef Impl::ViewOffsetRange< SubArg3_type > R3 ; + + // 'assign_subview' returns whether the subview offset_map + // introduces noncontiguity in the view. + const bool introduce_noncontiguity = + m_offset_map.assign_subview( src.m_offset_map + , R0::dimension( src.m_offset_map.N0 , arg0 ) + , R1::dimension( src.m_offset_map.N1 , arg1 ) + , R2::dimension( src.m_offset_map.N2 , arg2 ) + , R3::dimension( src.m_offset_map.N3 , arg3 ) + , 0 + , 0 + , 0 + , 0 + ); + + if ( m_offset_map.capacity() ) { + + m_management = src.m_management ; + + if ( introduce_noncontiguity ) m_management.set_noncontiguous(); + + m_ptr_on_device = src.m_ptr_on_device + + src.m_offset_map( R0::begin( arg0 ) + , R1::begin( arg1 ) + , R2::begin( arg2 ) + , R3::begin( arg3 ) + , 0 + , 0 + , 0 + , 0 ); + m_management.increment( m_ptr_on_device ); + } + } +} + +// Construct subview of a Rank 3 view +template< class DstDataType , class DstArg1Type , class DstArg2Type , class DstArg3Type > +template< class SrcDataType , class SrcArg1Type , class SrcArg2Type , class SrcArg3Type + , class SubArg0_type , class SubArg1_type , class SubArg2_type + > +KOKKOS_INLINE_FUNCTION +View< DstDataType , DstArg1Type , DstArg2Type , DstArg3Type , Impl::ViewDefault >:: +View( const View< SrcDataType , SrcArg1Type , SrcArg2Type , SrcArg3Type , Impl::ViewDefault > & src + , const SubArg0_type & arg0 + , const SubArg1_type & arg1 + , const SubArg2_type & arg2 + ) + : m_ptr_on_device( (typename traits::value_type*) NULL) + , m_offset_map() + , m_management() +{ + // This constructor can only be used to construct a subview + // from the source view. This type must match the subview type + // deduced from the source view and subview arguments. + + typedef Impl::ViewSubview< View< SrcDataType , SrcArg1Type , SrcArg2Type , SrcArg3Type , Impl::ViewDefault > + , SubArg0_type , SubArg1_type , SubArg2_type , void , void , void , void , void > + ViewSubviewDeduction ; + + enum { is_a_valid_subview_constructor = + Impl::StaticAssert< + Impl::is_same< View , typename ViewSubviewDeduction::type >::value + >::value + }; + + if ( is_a_valid_subview_constructor ) { + + typedef Impl::ViewOffsetRange< SubArg0_type > R0 ; + typedef Impl::ViewOffsetRange< SubArg1_type > R1 ; + typedef Impl::ViewOffsetRange< SubArg2_type > R2 ; + + // 'assign_subview' returns whether the subview offset_map + // introduces noncontiguity in the view. + const bool introduce_noncontiguity = + m_offset_map.assign_subview( src.m_offset_map + , R0::dimension( src.m_offset_map.N0 , arg0 ) + , R1::dimension( src.m_offset_map.N1 , arg1 ) + , R2::dimension( src.m_offset_map.N2 , arg2 ) + , 0 , 0 , 0 , 0 , 0); + + if ( m_offset_map.capacity() ) { + + m_management = src.m_management ; + + if ( introduce_noncontiguity ) m_management.set_noncontiguous(); + + m_ptr_on_device = src.m_ptr_on_device + + src.m_offset_map( R0::begin( arg0 ) + , R1::begin( arg1 ) + , R2::begin( arg2 ) + , 0 , 0 , 0 , 0 , 0 ); + m_management.increment( m_ptr_on_device ); + } + } +} + +// Construct subview of a Rank 2 view +template< class DstDataType , class DstArg1Type , class DstArg2Type , class DstArg3Type > +template< class SrcDataType , class SrcArg1Type , class SrcArg2Type , class SrcArg3Type + , class SubArg0_type , class SubArg1_type + > +KOKKOS_INLINE_FUNCTION +View< DstDataType , DstArg1Type , DstArg2Type , DstArg3Type , Impl::ViewDefault >:: +View( const View< SrcDataType , SrcArg1Type , SrcArg2Type , SrcArg3Type , Impl::ViewDefault > & src + , const SubArg0_type & arg0 + , const SubArg1_type & arg1 + ) + : m_ptr_on_device( (typename traits::value_type*) NULL) + , m_offset_map() + , m_management() +{ + // This constructor can only be used to construct a subview + // from the source view. This type must match the subview type + // deduced from the source view and subview arguments. + + typedef Impl::ViewSubview< View< SrcDataType , SrcArg1Type , SrcArg2Type , SrcArg3Type , Impl::ViewDefault > + , SubArg0_type , SubArg1_type , void , void , void , void , void , void > + ViewSubviewDeduction ; + + enum { is_a_valid_subview_constructor = + Impl::StaticAssert< + Impl::is_same< View , typename ViewSubviewDeduction::type >::value + >::value + }; + + if ( is_a_valid_subview_constructor ) { + + typedef Impl::ViewOffsetRange< SubArg0_type > R0 ; + typedef Impl::ViewOffsetRange< SubArg1_type > R1 ; + + // 'assign_subview' returns whether the subview offset_map + // introduces noncontiguity in the view. + const bool introduce_noncontiguity = + m_offset_map.assign_subview( src.m_offset_map + , R0::dimension( src.m_offset_map.N0 , arg0 ) + , R1::dimension( src.m_offset_map.N1 , arg1 ) + , 0 , 0 , 0 , 0 , 0 , 0 ); + + if ( m_offset_map.capacity() ) { + + m_management = src.m_management ; + + if ( introduce_noncontiguity ) m_management.set_noncontiguous(); + + m_ptr_on_device = src.m_ptr_on_device + + src.m_offset_map( R0::begin( arg0 ) + , R1::begin( arg1 ) + , 0 , 0 , 0 , 0 , 0 , 0 ); + m_management.increment( m_ptr_on_device ); + } + } +} + +// Construct subview of a Rank 1 view +template< class DstDataType , class DstArg1Type , class DstArg2Type , class DstArg3Type > +template< class SrcDataType , class SrcArg1Type , class SrcArg2Type , class SrcArg3Type + , class SubArg0_type + > +KOKKOS_INLINE_FUNCTION +View< DstDataType , DstArg1Type , DstArg2Type , DstArg3Type , Impl::ViewDefault >:: +View( const View< SrcDataType , SrcArg1Type , SrcArg2Type , SrcArg3Type , Impl::ViewDefault > & src + , const SubArg0_type & arg0 + ) + : m_ptr_on_device( (typename traits::value_type*) NULL) + , m_offset_map() + , m_management() +{ + // This constructor can only be used to construct a subview + // from the source view. This type must match the subview type + // deduced from the source view and subview arguments. + + typedef Impl::ViewSubview< View< SrcDataType , SrcArg1Type , SrcArg2Type , SrcArg3Type , Impl::ViewDefault > + , SubArg0_type , void , void , void , void , void , void , void > + ViewSubviewDeduction ; + + enum { is_a_valid_subview_constructor = + Impl::StaticAssert< + Impl::is_same< View , typename ViewSubviewDeduction::type >::value + >::value + }; + + if ( is_a_valid_subview_constructor ) { + + typedef Impl::ViewOffsetRange< SubArg0_type > R0 ; + + // 'assign_subview' returns whether the subview offset_map + // introduces noncontiguity in the view. + const bool introduce_noncontiguity = + m_offset_map.assign_subview( src.m_offset_map + , R0::dimension( src.m_offset_map.N0 , arg0 ) + , 0 , 0 , 0 , 0 , 0 , 0 , 0 ); + + if ( m_offset_map.capacity() ) { + + m_management = src.m_management ; + + if ( introduce_noncontiguity ) m_management.set_noncontiguous(); + + m_ptr_on_device = src.m_ptr_on_device + + src.m_offset_map( R0::begin( arg0 ) + , 0 , 0 , 0 , 0 , 0 , 0 , 0 ); + m_management.increment( m_ptr_on_device ); + } + } +} + +} /* namespace Kokkos */ + +//---------------------------------------------------------------------------- +//---------------------------------------------------------------------------- + +#endif /* #ifndef KOKKOS_VIEWDEFAULT_HPP */ + diff --git a/lib/kokkos/core/src/impl/Kokkos_ViewOffset.hpp b/lib/kokkos/core/src/impl/Kokkos_ViewOffset.hpp new file mode 100755 index 0000000000..1cced4954c --- /dev/null +++ b/lib/kokkos/core/src/impl/Kokkos_ViewOffset.hpp @@ -0,0 +1,1335 @@ +/* +//@HEADER +// ************************************************************************ +// +// Kokkos: Manycore Performance-Portable Multidimensional Arrays +// Copyright (2012) Sandia Corporation +// +// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +// the U.S. Government retains certain rights in this software. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// 1. Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// 3. Neither the name of the Corporation nor the names of the +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Questions? Contact H. Carter Edwards (hcedwar@sandia.gov) +// +// ************************************************************************ +//@HEADER +*/ + +#ifndef KOKKOS_VIEWOFFSET_HPP +#define KOKKOS_VIEWOFFSET_HPP + +#include +#include +#include +#include + +//---------------------------------------------------------------------------- +//---------------------------------------------------------------------------- + +namespace Kokkos { +struct ALL ; +} // namespace Kokkos + +//---------------------------------------------------------------------------- +//---------------------------------------------------------------------------- + +namespace Kokkos { namespace Impl { + +template < class ShapeType , class LayoutType , typename Enable = void > +struct ViewOffset ; + +//---------------------------------------------------------------------------- +// LayoutLeft AND ( 1 >= rank OR 0 == rank_dynamic ) : no padding / striding +template < class ShapeType > +struct ViewOffset< ShapeType , LayoutLeft + , typename enable_if<( 1 >= ShapeType::rank + || + 0 == ShapeType::rank_dynamic + )>::type > + : public ShapeType +{ + typedef size_t size_type ; + typedef ShapeType shape_type ; + typedef LayoutLeft array_layout ; + + enum { has_padding = false }; + + template< unsigned R > + KOKKOS_INLINE_FUNCTION + void assign( size_t n ) + { assign_shape_dimension( *this , n ); } + + // Return whether the subview introduced noncontiguity + template< class S , class L > + KOKKOS_INLINE_FUNCTION + typename Impl::enable_if<( 0 == shape_type::rank && + Impl::is_same::value + ), bool >::type + assign_subview( const ViewOffset & + , const size_t n0 + , const size_t n1 + , const size_t n2 + , const size_t n3 + , const size_t n4 + , const size_t n5 + , const size_t n6 + , const size_t n7 + ) + { + return false ; // did not introduce noncontiguity + } + + // This subview must be 1 == rank and 1 == rank_dynamic. + // The source dimension #0 must be non-zero and all other dimensions are zero. + // Return whether the subview introduced noncontiguity + template< class S , class L > + KOKKOS_INLINE_FUNCTION + typename Impl::enable_if<( 1 == shape_type::rank && + 1 == shape_type::rank_dynamic && + 1 <= S::rank && + Impl::is_same::value + ), bool >::type + assign_subview( const ViewOffset & + , const size_t n0 + , const size_t n1 + , const size_t n2 + , const size_t n3 + , const size_t n4 + , const size_t n5 + , const size_t n6 + , const size_t n7 + ) + { + // n1 .. n7 must be zero + shape_type::N0 = n0 ; + return false ; // did not introduce noncontiguity + } + + + KOKKOS_INLINE_FUNCTION + void assign( size_t n0 , unsigned n1 , unsigned n2 , unsigned n3 + , unsigned n4 , unsigned n5 , unsigned n6 , unsigned n7 + , unsigned = 0 ) + { shape_type::assign( *this , n0, n1, n2, n3, n4, n5, n6, n7 ); } + + template< class ShapeRHS > + KOKKOS_INLINE_FUNCTION + void assign( const ViewOffset< ShapeRHS , LayoutLeft > & rhs + , typename enable_if<( int(ShapeRHS::rank) == int(shape_type::rank) + && + int(ShapeRHS::rank_dynamic) <= int(shape_type::rank_dynamic) + )>::type * = 0 ) + { shape_type::assign( *this , rhs.N0, rhs.N1, rhs.N2, rhs.N3, rhs.N4, rhs.N5, rhs.N6, rhs.N7 ); } + + template< class ShapeRHS > + KOKKOS_INLINE_FUNCTION + void assign( const ViewOffset< ShapeRHS , LayoutRight > & rhs + , typename enable_if<( 1 == int(ShapeRHS::rank) + && + 1 == int(shape_type::rank) + && + 1 == int(shape_type::rank_dynamic) + )>::type * = 0 ) + { shape_type::assign( *this , rhs.N0, rhs.N1, rhs.N2, rhs.N3, rhs.N4, rhs.N5, rhs.N6, rhs.N7 ); } + + KOKKOS_INLINE_FUNCTION + void set_padding() {} + + KOKKOS_INLINE_FUNCTION + size_type cardinality() const + { return size_type(shape_type::N0) * shape_type::N1 * shape_type::N2 * shape_type::N3 * shape_type::N4 * shape_type::N5 * shape_type::N6 * shape_type::N7 ; } + + KOKKOS_INLINE_FUNCTION + size_type capacity() const + { return size_type(shape_type::N0) * shape_type::N1 * shape_type::N2 * shape_type::N3 * shape_type::N4 * shape_type::N5 * shape_type::N6 * shape_type::N7 ; } + + // Stride with [ rank ] value is the total length + template< typename iType > + KOKKOS_INLINE_FUNCTION + void stride( iType * const s ) const + { + s[0] = 1 ; + if ( 0 < shape_type::rank ) { s[1] = shape_type::N0 ; } + if ( 1 < shape_type::rank ) { s[2] = s[1] * shape_type::N1 ; } + if ( 2 < shape_type::rank ) { s[3] = s[2] * shape_type::N2 ; } + if ( 3 < shape_type::rank ) { s[4] = s[3] * shape_type::N3 ; } + if ( 4 < shape_type::rank ) { s[5] = s[4] * shape_type::N4 ; } + if ( 5 < shape_type::rank ) { s[6] = s[5] * shape_type::N5 ; } + if ( 6 < shape_type::rank ) { s[7] = s[6] * shape_type::N6 ; } + if ( 7 < shape_type::rank ) { s[8] = s[7] * shape_type::N7 ; } + } + + KOKKOS_INLINE_FUNCTION size_type stride_0() const { return 1 ; } + KOKKOS_INLINE_FUNCTION size_type stride_1() const { return shape_type::N0 ; } + KOKKOS_INLINE_FUNCTION size_type stride_2() const { return shape_type::N0 * shape_type::N1 ; } + KOKKOS_INLINE_FUNCTION size_type stride_3() const { return shape_type::N0 * shape_type::N1 * shape_type::N2 ; } + + KOKKOS_INLINE_FUNCTION + size_type stride_4() const + { return shape_type::N0 * shape_type::N1 * shape_type::N2 * shape_type::N3 ; } + + KOKKOS_INLINE_FUNCTION + size_type stride_5() const + { return shape_type::N0 * shape_type::N1 * shape_type::N2 * shape_type::N3 * shape_type::N4 ; } + + KOKKOS_INLINE_FUNCTION + size_type stride_6() const + { return shape_type::N0 * shape_type::N1 * shape_type::N2 * shape_type::N3 * shape_type::N4 * shape_type::N5 ; } + + KOKKOS_INLINE_FUNCTION + size_type stride_7() const + { return shape_type::N0 * shape_type::N1 * shape_type::N2 * shape_type::N3 * shape_type::N4 * shape_type::N5 * shape_type::N6 ; } + + // rank 1 + template< typename I0 > + KOKKOS_FORCEINLINE_FUNCTION + size_type operator()( I0 const & i0 ) const { return i0 ; } + + // rank 2 + template < typename I0 , typename I1 > + KOKKOS_FORCEINLINE_FUNCTION + size_type operator()( I0 const & i0 , I1 const & i1 ) const + { return i0 + shape_type::N0 * i1 ; } + + //rank 3 + template + KOKKOS_FORCEINLINE_FUNCTION + size_type operator()( I0 const& i0 + , I1 const& i1 + , I2 const& i2 + ) const + { + return i0 + shape_type::N0 * ( + i1 + shape_type::N1 * i2 ); + } + + //rank 4 + template + KOKKOS_FORCEINLINE_FUNCTION + size_type operator()( I0 const& i0, I1 const& i1, I2 const& i2, I3 const& i3 ) const + { + return i0 + shape_type::N0 * ( + i1 + shape_type::N1 * ( + i2 + shape_type::N2 * i3 )); + } + + //rank 5 + template < typename I0, typename I1, typename I2, typename I3 + ,typename I4 > + KOKKOS_FORCEINLINE_FUNCTION + size_type operator()( I0 const& i0, I1 const& i1, I2 const& i2, I3 const& i3, I4 const& i4 ) const + { + return i0 + shape_type::N0 * ( + i1 + shape_type::N1 * ( + i2 + shape_type::N2 * ( + i3 + shape_type::N3 * i4 ))); + } + + //rank 6 + template < typename I0, typename I1, typename I2, typename I3 + ,typename I4, typename I5 > + KOKKOS_FORCEINLINE_FUNCTION + size_type operator()( I0 const& i0, I1 const& i1, I2 const& i2, I3 const& i3, I4 const& i4, I5 const& i5 ) const + { + return i0 + shape_type::N0 * ( + i1 + shape_type::N1 * ( + i2 + shape_type::N2 * ( + i3 + shape_type::N3 * ( + i4 + shape_type::N4 * i5 )))); + } + + //rank 7 + template < typename I0, typename I1, typename I2, typename I3 + ,typename I4, typename I5, typename I6 > + KOKKOS_FORCEINLINE_FUNCTION + size_type operator()( I0 const& i0, I1 const& i1, I2 const& i2, I3 const& i3, I4 const& i4, I5 const& i5, I6 const& i6) const + { + return i0 + shape_type::N0 * ( + i1 + shape_type::N1 * ( + i2 + shape_type::N2 * ( + i3 + shape_type::N3 * ( + i4 + shape_type::N4 * ( + i5 + shape_type::N5 * i6 ))))); + } + + //rank 8 + template < typename I0, typename I1, typename I2, typename I3 + ,typename I4, typename I5, typename I6, typename I7 > + KOKKOS_FORCEINLINE_FUNCTION + size_type operator()( I0 const& i0, I1 const& i1, I2 const& i2, I3 const& i3, I4 const& i4, I5 const& i5, I6 const& i6, I7 const& i7) const + { + return i0 + shape_type::N0 * ( + i1 + shape_type::N1 * ( + i2 + shape_type::N2 * ( + i3 + shape_type::N3 * ( + i4 + shape_type::N4 * ( + i5 + shape_type::N5 * ( + i6 + shape_type::N6 * i7 )))))); + } +}; + +//---------------------------------------------------------------------------- +// LayoutLeft AND ( 1 < rank AND 0 < rank_dynamic ) : has padding / striding +template < class ShapeType > +struct ViewOffset< ShapeType , LayoutLeft + , typename enable_if<( 1 < ShapeType::rank + && + 0 < ShapeType::rank_dynamic + )>::type > + : public ShapeType +{ + typedef size_t size_type ; + typedef ShapeType shape_type ; + typedef LayoutLeft array_layout ; + + enum { has_padding = true }; + + size_type S0 ; + + // This subview must be 2 == rank and 2 == rank_dynamic + // due to only having stride #0. + // The source dimension #0 must be non-zero for stride-one leading dimension. + // If source is rank deficient then set to zero. + // Return whether the subview introduced noncontiguity + template< class S , class L > + KOKKOS_INLINE_FUNCTION + typename Impl::enable_if<( 2 == shape_type::rank && + 2 == shape_type::rank_dynamic && + 2 <= S::rank && + Impl::is_same::value + ), bool >::type + assign_subview( const ViewOffset & rhs + , const size_t n0 + , const size_t n1 + , const size_t n2 + , const size_t n3 + , const size_t n4 + , const size_t n5 + , const size_t n6 + , const size_t n7 + ) + { + // N0 = n0 ; + // N1 = second non-zero dimension + // S0 = stride for second non-zero dimension + shape_type::N0 = 0 ; + shape_type::N1 = 0 ; + S0 = 0 ; + + if ( 0 == n0 ) {} + else if ( n1 ) { shape_type::N0 = n0 ; shape_type::N1 = n1 ; S0 = rhs.stride_1(); } + else if ( 2 < S::rank && n2 ) { shape_type::N0 = n0 ; shape_type::N1 = n2 ; S0 = rhs.stride_2(); } + else if ( 3 < S::rank && n3 ) { shape_type::N0 = n0 ; shape_type::N1 = n3 ; S0 = rhs.stride_3(); } + else if ( 4 < S::rank && n4 ) { shape_type::N0 = n0 ; shape_type::N1 = n4 ; S0 = rhs.stride_4(); } + else if ( 5 < S::rank && n5 ) { shape_type::N0 = n0 ; shape_type::N1 = n5 ; S0 = rhs.stride_5(); } + else if ( 6 < S::rank && n6 ) { shape_type::N0 = n0 ; shape_type::N1 = n6 ; S0 = rhs.stride_6(); } + else if ( 7 < S::rank && n7 ) { shape_type::N0 = n0 ; shape_type::N1 = n7 ; S0 = rhs.stride_7(); } + + // Introduce noncontiguity if change the first dimension + // or took a range of a dimension after the second. + return ( size_t(shape_type::N0) != size_t(rhs.N0) ) || ( 0 == n1 ); + } + + + template< unsigned R > + KOKKOS_INLINE_FUNCTION + void assign( size_t n ) + { assign_shape_dimension( *this , n ); } + + + KOKKOS_INLINE_FUNCTION + void assign( size_t n0 , unsigned n1 , unsigned n2 , unsigned n3 + , unsigned n4 , unsigned n5 , unsigned n6 , unsigned n7 + , unsigned = 0 ) + { shape_type::assign( *this , n0, n1, n2, n3, n4, n5, n6, n7 ); S0 = shape_type::N0 ; } + + template< class ShapeRHS > + KOKKOS_INLINE_FUNCTION + void assign( const ViewOffset< ShapeRHS , LayoutLeft > & rhs + , typename enable_if<( int(ShapeRHS::rank) == int(shape_type::rank) + && + int(ShapeRHS::rank_dynamic) <= int(shape_type::rank_dynamic) + && + int(ShapeRHS::rank_dynamic) == 0 + )>::type * = 0 ) + { + shape_type::assign( *this , rhs.N0, rhs.N1, rhs.N2, rhs.N3, rhs.N4, rhs.N5, rhs.N6, rhs.N7 ); + S0 = shape_type::N0 ; // No padding when dynamic_rank == 0 + } + + template< class ShapeRHS > + KOKKOS_INLINE_FUNCTION + void assign( const ViewOffset< ShapeRHS , LayoutLeft > & rhs + , typename enable_if<( int(ShapeRHS::rank) == int(shape_type::rank) + && + int(ShapeRHS::rank_dynamic) <= int(shape_type::rank_dynamic) + && + int(ShapeRHS::rank_dynamic) > 0 + )>::type * = 0 ) + { + shape_type::assign( *this , rhs.N0, rhs.N1, rhs.N2, rhs.N3, rhs.N4, rhs.N5, rhs.N6, rhs.N7 ); + S0 = rhs.S0 ; // possibly padding when dynamic rank > 0 + } + + KOKKOS_INLINE_FUNCTION + void set_padding() + { + enum { div = MEMORY_ALIGNMENT / shape_type::scalar_size }; + enum { mod = MEMORY_ALIGNMENT % shape_type::scalar_size }; + enum { align = 0 == mod ? div : 0 }; + + if ( align && MEMORY_ALIGNMENT_THRESHOLD * align < S0 ) { + + const size_type count_mod = S0 % ( div ? div : 1 ); + + if ( count_mod ) { S0 += align - count_mod ; } + } + } + + KOKKOS_INLINE_FUNCTION + size_type cardinality() const + { return size_type(shape_type::N0) * shape_type::N1 * shape_type::N2 * shape_type::N3 * shape_type::N4 * shape_type::N5 * shape_type::N6 * shape_type::N7 ; } + + KOKKOS_INLINE_FUNCTION + size_type capacity() const + { return size_type(S0) * shape_type::N1 * shape_type::N2 * shape_type::N3 * shape_type::N4 * shape_type::N5 * shape_type::N6 * shape_type::N7 ; } + + // Stride with [ rank ] as total length + template< typename iType > + KOKKOS_INLINE_FUNCTION + void stride( iType * const s ) const + { + s[0] = 1 ; + if ( 0 < shape_type::rank ) { s[1] = S0 ; } + if ( 1 < shape_type::rank ) { s[2] = s[1] * shape_type::N1 ; } + if ( 2 < shape_type::rank ) { s[3] = s[2] * shape_type::N2 ; } + if ( 3 < shape_type::rank ) { s[4] = s[3] * shape_type::N3 ; } + if ( 4 < shape_type::rank ) { s[5] = s[4] * shape_type::N4 ; } + if ( 5 < shape_type::rank ) { s[6] = s[5] * shape_type::N5 ; } + if ( 6 < shape_type::rank ) { s[7] = s[6] * shape_type::N6 ; } + if ( 7 < shape_type::rank ) { s[8] = s[7] * shape_type::N6 ; } + } + + KOKKOS_INLINE_FUNCTION size_type stride_0() const { return 1 ; } + KOKKOS_INLINE_FUNCTION size_type stride_1() const { return S0 ; } + KOKKOS_INLINE_FUNCTION size_type stride_2() const { return S0 * shape_type::N1 ; } + KOKKOS_INLINE_FUNCTION size_type stride_3() const { return S0 * shape_type::N1 * shape_type::N2 ; } + + KOKKOS_INLINE_FUNCTION + size_type stride_4() const + { return S0 * shape_type::N1 * shape_type::N2 * shape_type::N3 ; } + + KOKKOS_INLINE_FUNCTION + size_type stride_5() const + { return S0 * shape_type::N1 * shape_type::N2 * shape_type::N3 * shape_type::N4 ; } + + KOKKOS_INLINE_FUNCTION + size_type stride_6() const + { return S0 * shape_type::N1 * shape_type::N2 * shape_type::N3 * shape_type::N4 * shape_type::N5 ; } + + KOKKOS_INLINE_FUNCTION + size_type stride_7() const + { return S0 * shape_type::N1 * shape_type::N2 * shape_type::N3 * shape_type::N4 * shape_type::N5 * shape_type::N6 ; } + + // rank 2 + template < typename I0 , typename I1 > + KOKKOS_FORCEINLINE_FUNCTION + size_type operator()( I0 const & i0 , I1 const & i1) const + { return i0 + S0 * i1 ; } + + //rank 3 + template + KOKKOS_FORCEINLINE_FUNCTION + size_type operator()( I0 const& i0, I1 const& i1, I2 const& i2 ) const + { + return i0 + S0 * ( + i1 + shape_type::N1 * i2 ); + } + + //rank 4 + template + KOKKOS_FORCEINLINE_FUNCTION + size_type operator()( I0 const& i0, I1 const& i1, I2 const& i2, I3 const& i3 ) const + { + return i0 + S0 * ( + i1 + shape_type::N1 * ( + i2 + shape_type::N2 * i3 )); + } + + //rank 5 + template < typename I0, typename I1, typename I2, typename I3 + ,typename I4 > + KOKKOS_FORCEINLINE_FUNCTION + size_type operator()( I0 const& i0, I1 const& i1, I2 const& i2, I3 const& i3, I4 const& i4 ) const + { + return i0 + S0 * ( + i1 + shape_type::N1 * ( + i2 + shape_type::N2 * ( + i3 + shape_type::N3 * i4 ))); + } + + //rank 6 + template < typename I0, typename I1, typename I2, typename I3 + ,typename I4, typename I5 > + KOKKOS_FORCEINLINE_FUNCTION + size_type operator()( I0 const& i0, I1 const& i1, I2 const& i2, I3 const& i3, I4 const& i4, I5 const& i5 ) const + { + return i0 + S0 * ( + i1 + shape_type::N1 * ( + i2 + shape_type::N2 * ( + i3 + shape_type::N3 * ( + i4 + shape_type::N4 * i5 )))); + } + + //rank 7 + template < typename I0, typename I1, typename I2, typename I3 + ,typename I4, typename I5, typename I6 > + KOKKOS_FORCEINLINE_FUNCTION + size_type operator()( I0 const& i0, I1 const& i1, I2 const& i2, I3 const& i3, I4 const& i4, I5 const& i5, I6 const& i6 ) const + { + return i0 + S0 * ( + i1 + shape_type::N1 * ( + i2 + shape_type::N2 * ( + i3 + shape_type::N3 * ( + i4 + shape_type::N4 * ( + i5 + shape_type::N5 * i6 ))))); + } + + //rank 8 + template < typename I0, typename I1, typename I2, typename I3 + ,typename I4, typename I5, typename I6, typename I7 > + KOKKOS_FORCEINLINE_FUNCTION + size_type operator()( I0 const& i0, I1 const& i1, I2 const& i2, I3 const& i3, I4 const& i4, I5 const& i5, I6 const& i6, I7 const& i7 ) const + { + return i0 + S0 * ( + i1 + shape_type::N1 * ( + i2 + shape_type::N2 * ( + i3 + shape_type::N3 * ( + i4 + shape_type::N4 * ( + i5 + shape_type::N5 * ( + i6 + shape_type::N6 * i7 )))))); + } +}; + +//---------------------------------------------------------------------------- +// LayoutRight AND ( 1 >= rank OR 1 >= rank_dynamic ) : no padding / striding +template < class ShapeType > +struct ViewOffset< ShapeType , LayoutRight + , typename enable_if<( 1 >= ShapeType::rank + || + 1 >= ShapeType::rank_dynamic + )>::type > + : public ShapeType +{ + typedef size_t size_type; + typedef ShapeType shape_type; + typedef LayoutRight array_layout ; + + enum { has_padding = false }; + + // This subview must be 1 == rank and 1 == rank_dynamic + // The source view's last dimension must be non-zero + // Return whether the subview introduced noncontiguity + template< class S , class L > + KOKKOS_INLINE_FUNCTION + typename Impl::enable_if<( 0 == shape_type::rank && + Impl::is_same::value + ), bool >::type + assign_subview( const ViewOffset & + , const size_t n0 + , const size_t n1 + , const size_t n2 + , const size_t n3 + , const size_t n4 + , const size_t n5 + , const size_t n6 + , const size_t n7 + ) + { return false ; } + + // This subview must be 1 == rank and 1 == rank_dynamic + // The source view's last dimension must be non-zero + // Return whether the subview introduced noncontiguity + template< class S , class L > + KOKKOS_INLINE_FUNCTION + typename Impl::enable_if<( 1 == shape_type::rank && + 1 == shape_type::rank_dynamic && + 1 <= S::rank && + Impl::is_same::value + ), bool >::type + assign_subview( const ViewOffset & + , const size_t n0 + , const size_t n1 + , const size_t n2 + , const size_t n3 + , const size_t n4 + , const size_t n5 + , const size_t n6 + , const size_t n7 + ) + { + shape_type::N0 = S::rank == 1 ? n0 : ( + S::rank == 2 ? n1 : ( + S::rank == 3 ? n2 : ( + S::rank == 4 ? n3 : ( + S::rank == 5 ? n4 : ( + S::rank == 6 ? n5 : ( + S::rank == 7 ? n6 : n7 )))))); + // should have n0 .. n_(rank-2) equal zero + return false ; + } + + template< unsigned R > + KOKKOS_INLINE_FUNCTION + void assign( unsigned n ) + { assign_shape_dimension( *this , n ); } + + KOKKOS_INLINE_FUNCTION + void assign( unsigned n0 , unsigned n1 , unsigned n2 , unsigned n3 + , unsigned n4 , unsigned n5 , unsigned n6 , unsigned n7 + , unsigned = 0 ) + { shape_type::assign( *this , n0, n1, n2, n3, n4, n5, n6, n7 ); } + + template< class ShapeRHS > + KOKKOS_INLINE_FUNCTION + void assign( const ViewOffset< ShapeRHS , LayoutRight > & rhs + , typename enable_if<( int(ShapeRHS::rank) == int(shape_type::rank) + && + int(ShapeRHS::rank_dynamic) <= int(shape_type::rank_dynamic) + )>::type * = 0 ) + { shape_type::assign( *this , rhs.N0, rhs.N1, rhs.N2, rhs.N3, rhs.N4, rhs.N5, rhs.N6, rhs.N7 ); } + + template< class ShapeRHS > + KOKKOS_INLINE_FUNCTION + void assign( const ViewOffset< ShapeRHS , LayoutLeft > & rhs + , typename enable_if<( 1 == int(ShapeRHS::rank) + && + 1 == int(shape_type::rank) + && + 1 == int(shape_type::rank_dynamic) + )>::type * = 0 ) + { shape_type::assign( *this , rhs.N0, rhs.N1, rhs.N2, rhs.N3, rhs.N4, rhs.N5, rhs.N6, rhs.N7 ); } + + KOKKOS_INLINE_FUNCTION + void set_padding() {} + + KOKKOS_INLINE_FUNCTION + size_type cardinality() const + { return size_type(shape_type::N0) * shape_type::N1 * shape_type::N2 * shape_type::N3 * shape_type::N4 * shape_type::N5 * shape_type::N6 * shape_type::N7 ; } + + KOKKOS_INLINE_FUNCTION + size_type capacity() const + { return size_type(shape_type::N0) * shape_type::N1 * shape_type::N2 * shape_type::N3 * shape_type::N4 * shape_type::N5 * shape_type::N6 * shape_type::N7 ; } + + size_type stride_R() const + { + return size_type(shape_type::N1) * shape_type::N2 * shape_type::N3 * + shape_type::N4 * shape_type::N5 * shape_type::N6 * shape_type::N7 ; + }; + + // Stride with [rank] as total length + template< typename iType > + KOKKOS_INLINE_FUNCTION + void stride( iType * const s ) const + { + size_type n = 1 ; + if ( 7 < shape_type::rank ) { s[7] = n ; n *= shape_type::N7 ; } + if ( 6 < shape_type::rank ) { s[6] = n ; n *= shape_type::N6 ; } + if ( 5 < shape_type::rank ) { s[5] = n ; n *= shape_type::N5 ; } + if ( 4 < shape_type::rank ) { s[4] = n ; n *= shape_type::N4 ; } + if ( 3 < shape_type::rank ) { s[3] = n ; n *= shape_type::N3 ; } + if ( 2 < shape_type::rank ) { s[2] = n ; n *= shape_type::N2 ; } + if ( 1 < shape_type::rank ) { s[1] = n ; n *= shape_type::N1 ; } + if ( 0 < shape_type::rank ) { s[0] = n ; } + s[shape_type::rank] = n * shape_type::N0 ; + } + + KOKKOS_INLINE_FUNCTION + size_type stride_7() const { return 1 ; } + + KOKKOS_INLINE_FUNCTION + size_type stride_6() const { return shape_type::N7 ; } + + KOKKOS_INLINE_FUNCTION + size_type stride_5() const { return shape_type::N7 * shape_type::N6 ; } + + KOKKOS_INLINE_FUNCTION + size_type stride_4() const { return shape_type::N7 * shape_type::N6 * shape_type::N5 ; } + + KOKKOS_INLINE_FUNCTION + size_type stride_3() const { return shape_type::N7 * shape_type::N6 * shape_type::N5 * shape_type::N4 ; } + + KOKKOS_INLINE_FUNCTION + size_type stride_2() const { return shape_type::N7 * shape_type::N6 * shape_type::N5 * shape_type::N4 * shape_type::N3 ; } + + KOKKOS_INLINE_FUNCTION + size_type stride_1() const { return shape_type::N7 * shape_type::N6 * shape_type::N5 * shape_type::N4 * shape_type::N3 * shape_type::N2 ; } + + KOKKOS_INLINE_FUNCTION + size_type stride_0() const { return shape_type::N7 * shape_type::N6 * shape_type::N5 * shape_type::N4 * shape_type::N3 * shape_type::N2 * shape_type::N1 ; } + + // rank 2 + template + KOKKOS_FORCEINLINE_FUNCTION + size_type operator()( I0 const& i0, I1 const& i1 ) const + { + return i1 + shape_type::N1 * i0 ; + } + + template + KOKKOS_FORCEINLINE_FUNCTION + size_type operator()( I0 const& i0, I1 const& i1, I2 const& i2 ) const + { + return i2 + shape_type::N2 * ( + i1 + shape_type::N1 * ( i0 )); + } + + template + KOKKOS_FORCEINLINE_FUNCTION + size_type operator()( I0 const& i0, I1 const& i1, I2 const& i2 , I3 const& i3 ) const + { + return i3 + shape_type::N3 * ( + i2 + shape_type::N2 * ( + i1 + shape_type::N1 * ( i0 ))); + } + + template < typename I0, typename I1, typename I2, typename I3 + ,typename I4 > + KOKKOS_FORCEINLINE_FUNCTION + size_type operator()( I0 const& i0, I1 const& i1, I2 const& i2 , I3 const& i3, I4 const& i4 ) const + { + return i4 + shape_type::N4 * ( + i3 + shape_type::N3 * ( + i2 + shape_type::N2 * ( + i1 + shape_type::N1 * ( i0 )))); + } + + template < typename I0, typename I1, typename I2, typename I3 + ,typename I4, typename I5 > + KOKKOS_FORCEINLINE_FUNCTION + size_type operator()( I0 const& i0, I1 const& i1, I2 const& i2 , I3 const& i3, I4 const& i4, I5 const& i5 ) const + { + return i5 + shape_type::N5 * ( + i4 + shape_type::N4 * ( + i3 + shape_type::N3 * ( + i2 + shape_type::N2 * ( + i1 + shape_type::N1 * ( i0 ))))); + } + + template < typename I0, typename I1, typename I2, typename I3 + ,typename I4, typename I5, typename I6 > + KOKKOS_FORCEINLINE_FUNCTION + size_type operator()( I0 const& i0, I1 const& i1, I2 const& i2 , I3 const& i3, I4 const& i4, I5 const& i5, I6 const& i6 ) const + { + return i6 + shape_type::N6 * ( + i5 + shape_type::N5 * ( + i4 + shape_type::N4 * ( + i3 + shape_type::N3 * ( + i2 + shape_type::N2 * ( + i1 + shape_type::N1 * ( i0 )))))); + } + + template < typename I0, typename I1, typename I2, typename I3 + ,typename I4, typename I5, typename I6, typename I7 > + KOKKOS_FORCEINLINE_FUNCTION + size_type operator()( I0 const& i0, I1 const& i1, I2 const& i2 , I3 const& i3, I4 const& i4, I5 const& i5, I6 const& i6, I7 const& i7 ) const + { + return i7 + shape_type::N7 * ( + i6 + shape_type::N6 * ( + i5 + shape_type::N5 * ( + i4 + shape_type::N4 * ( + i3 + shape_type::N3 * ( + i2 + shape_type::N2 * ( + i1 + shape_type::N1 * ( i0 ))))))); + } +}; + +//---------------------------------------------------------------------------- +// LayoutRight AND ( 1 < rank AND 1 < rank_dynamic ) : has padding / striding +template < class ShapeType > +struct ViewOffset< ShapeType , LayoutRight + , typename enable_if<( 1 < ShapeType::rank + && + 1 < ShapeType::rank_dynamic + )>::type > + : public ShapeType +{ + typedef size_t size_type; + typedef ShapeType shape_type; + typedef LayoutRight array_layout ; + + enum { has_padding = true }; + + size_type SR ; + + // This subview must be 2 == rank and 2 == rank_dynamic + // due to only having stride #(rank-1). + // The source dimension #(rank-1) must be non-zero for stride-one leading dimension. + // If source is rank deficient then set to zero. + // Return whether the subview introduced noncontiguity + template< class S , class L > + KOKKOS_INLINE_FUNCTION + typename Impl::enable_if<( 2 == shape_type::rank && + 2 == shape_type::rank_dynamic && + 2 <= S::rank && + Impl::is_same::value + ), bool >::type + assign_subview( const ViewOffset & rhs + , const size_t n0 + , const size_t n1 + , const size_t n2 + , const size_t n3 + , const size_t n4 + , const size_t n5 + , const size_t n6 + , const size_t n7 + ) + { + const size_type nR = S::rank == 2 ? n1 : ( + S::rank == 3 ? n2 : ( + S::rank == 4 ? n3 : ( + S::rank == 5 ? n4 : ( + S::rank == 6 ? n5 : ( + S::rank == 7 ? n6 : n7 ))))); + + // N0 = first non-zero-dimension + // N1 = last non-zero dimension + // SR = stride for second non-zero dimension + shape_type::N0 = 0 ; + shape_type::N1 = 0 ; + SR = 0 ; + + if ( 0 == nR ) {} + else if ( n0 ) { shape_type::N0 = n0 ; shape_type::N1 = nR ; SR = rhs.stride_0(); } + else if ( 2 < S::rank && n1 ) { shape_type::N0 = n1 ; shape_type::N1 = nR ; SR = rhs.stride_1(); } + else if ( 3 < S::rank && n2 ) { shape_type::N0 = n2 ; shape_type::N1 = nR ; SR = rhs.stride_2(); } + else if ( 4 < S::rank && n3 ) { shape_type::N0 = n3 ; shape_type::N1 = nR ; SR = rhs.stride_3(); } + else if ( 5 < S::rank && n4 ) { shape_type::N0 = n4 ; shape_type::N1 = nR ; SR = rhs.stride_4(); } + else if ( 6 < S::rank && n5 ) { shape_type::N0 = n5 ; shape_type::N1 = nR ; SR = rhs.stride_5(); } + else if ( 7 < S::rank && n6 ) { shape_type::N0 = n6 ; shape_type::N1 = nR ; SR = rhs.stride_6(); } + + // Introduce noncontiguous if change the last dimension + // or take a range of a dimension other than the second-to-last dimension. + + return 2 == S::rank ? ( size_t(shape_type::N1) != size_t(rhs.N1) || 0 == n0 ) : ( + 3 == S::rank ? ( size_t(shape_type::N1) != size_t(rhs.N2) || 0 == n1 ) : ( + 4 == S::rank ? ( size_t(shape_type::N1) != size_t(rhs.N3) || 0 == n2 ) : ( + 5 == S::rank ? ( size_t(shape_type::N1) != size_t(rhs.N4) || 0 == n3 ) : ( + 6 == S::rank ? ( size_t(shape_type::N1) != size_t(rhs.N5) || 0 == n4 ) : ( + 7 == S::rank ? ( size_t(shape_type::N1) != size_t(rhs.N6) || 0 == n5 ) : ( + ( size_t(shape_type::N1) != size_t(rhs.N7) || 0 == n6 ) )))))); + } + + template< unsigned R > + KOKKOS_INLINE_FUNCTION + void assign( unsigned n ) + { assign_shape_dimension( *this , n ); } + + KOKKOS_INLINE_FUNCTION + void assign( unsigned n0 , unsigned n1 , unsigned n2 , unsigned n3 + , unsigned n4 , unsigned n5 , unsigned n6 , unsigned n7 + , unsigned = 0 ) + { + shape_type::assign( *this , n0, n1, n2, n3, n4, n5, n6, n7 ); + SR = size_type(shape_type::N1) * shape_type::N2 * shape_type::N3 * shape_type::N4 * shape_type::N5 * shape_type::N6 * shape_type::N7 ; + } + + template< class ShapeRHS > + KOKKOS_INLINE_FUNCTION + void assign( const ViewOffset< ShapeRHS , LayoutRight > & rhs + , typename enable_if<( int(ShapeRHS::rank) == int(shape_type::rank) + && + int(ShapeRHS::rank_dynamic) <= int(shape_type::rank_dynamic) + && + int(ShapeRHS::rank_dynamic) <= 1 + )>::type * = 0 ) + { + shape_type::assign( *this , rhs.N0, rhs.N1, rhs.N2, rhs.N3, rhs.N4, rhs.N5, rhs.N6, rhs.N7 ); + SR = shape_type::N1 * shape_type::N2 * shape_type::N3 * shape_type::N4 * shape_type::N5 * shape_type::N6 * shape_type::N7 ; + } + + template< class ShapeRHS > + KOKKOS_INLINE_FUNCTION + void assign( const ViewOffset< ShapeRHS , LayoutRight > & rhs + , typename enable_if<( int(ShapeRHS::rank) == int(shape_type::rank) + && + int(ShapeRHS::rank_dynamic) <= int(shape_type::rank_dynamic) + && + int(ShapeRHS::rank_dynamic) > 1 + )>::type * = 0 ) + { + shape_type::assign( *this , rhs.N0, rhs.N1, rhs.N2, rhs.N3, rhs.N4, rhs.N5, rhs.N6, rhs.N7 ); + SR = rhs.SR ; + } + + KOKKOS_INLINE_FUNCTION + void set_padding() + { + enum { div = MEMORY_ALIGNMENT / shape_type::scalar_size }; + enum { mod = MEMORY_ALIGNMENT % shape_type::scalar_size }; + enum { align = 0 == mod ? div : 0 }; + + if ( align && MEMORY_ALIGNMENT_THRESHOLD * align < SR ) { + + const size_type count_mod = SR % ( div ? div : 1 ); + + if ( count_mod ) { SR += align - count_mod ; } + } + } + + KOKKOS_INLINE_FUNCTION + size_type cardinality() const + { return size_type(shape_type::N0) * shape_type::N1 * shape_type::N2 * shape_type::N3 * shape_type::N4 * shape_type::N5 * shape_type::N6 * shape_type::N7 ; } + + KOKKOS_INLINE_FUNCTION + size_type capacity() const { return shape_type::N0 * SR ; } + + template< typename iType > + KOKKOS_INLINE_FUNCTION + void stride( iType * const s ) const + { + size_type n = 1 ; + if ( 7 < shape_type::rank ) { s[7] = n ; n *= shape_type::N7 ; } + if ( 6 < shape_type::rank ) { s[6] = n ; n *= shape_type::N6 ; } + if ( 5 < shape_type::rank ) { s[5] = n ; n *= shape_type::N5 ; } + if ( 4 < shape_type::rank ) { s[4] = n ; n *= shape_type::N4 ; } + if ( 3 < shape_type::rank ) { s[3] = n ; n *= shape_type::N3 ; } + if ( 2 < shape_type::rank ) { s[2] = n ; n *= shape_type::N2 ; } + if ( 1 < shape_type::rank ) { s[1] = n ; n *= shape_type::N1 ; } + if ( 0 < shape_type::rank ) { s[0] = SR ; } + s[shape_type::rank] = SR * shape_type::N0 ; + } + + KOKKOS_INLINE_FUNCTION + size_type stride_7() const { return 1 ; } + + KOKKOS_INLINE_FUNCTION + size_type stride_6() const { return shape_type::N7 ; } + + KOKKOS_INLINE_FUNCTION + size_type stride_5() const { return shape_type::N7 * shape_type::N6 ; } + + KOKKOS_INLINE_FUNCTION + size_type stride_4() const { return shape_type::N7 * shape_type::N6 * shape_type::N5 ; } + + KOKKOS_INLINE_FUNCTION + size_type stride_3() const { return shape_type::N7 * shape_type::N6 * shape_type::N5 * shape_type::N4 ; } + + KOKKOS_INLINE_FUNCTION + size_type stride_2() const { return shape_type::N7 * shape_type::N6 * shape_type::N5 * shape_type::N4 * shape_type::N3 ; } + + KOKKOS_INLINE_FUNCTION + size_type stride_1() const { return shape_type::N7 * shape_type::N6 * shape_type::N5 * shape_type::N4 * shape_type::N3 * shape_type::N2 ; } + + KOKKOS_INLINE_FUNCTION + size_type stride_0() const { return SR ; } + + // rank 2 + template + KOKKOS_FORCEINLINE_FUNCTION + size_type operator()( I0 const& i0, I1 const& i1 ) const + { + return i1 + i0 * SR ; + } + + template + KOKKOS_FORCEINLINE_FUNCTION + size_type operator()( I0 const& i0, I1 const& i1, I2 const& i2 ) const + { + return i2 + shape_type::N2 * ( i1 ) + + i0 * SR ; + } + + template + KOKKOS_FORCEINLINE_FUNCTION + size_type operator()( I0 const& i0, I1 const& i1, I2 const& i2 , I3 const& i3 ) const + { + return i3 + shape_type::N3 * ( + i2 + shape_type::N2 * ( i1 )) + + i0 * SR ; + } + + template < typename I0, typename I1, typename I2, typename I3 + ,typename I4 > + KOKKOS_FORCEINLINE_FUNCTION + size_type operator()( I0 const& i0, I1 const& i1, I2 const& i2 , I3 const& i3, I4 const& i4 ) const + { + return i4 + shape_type::N4 * ( + i3 + shape_type::N3 * ( + i2 + shape_type::N2 * ( i1 ))) + + i0 * SR ; + } + + template < typename I0, typename I1, typename I2, typename I3 + ,typename I4, typename I5 > + KOKKOS_FORCEINLINE_FUNCTION + size_type operator()( I0 const& i0, I1 const& i1, I2 const& i2 , I3 const& i3, I4 const& i4, I5 const& i5 ) const + { + return i5 + shape_type::N5 * ( + i4 + shape_type::N4 * ( + i3 + shape_type::N3 * ( + i2 + shape_type::N2 * ( i1 )))) + + i0 * SR ; + } + + template < typename I0, typename I1, typename I2, typename I3 + ,typename I4, typename I5, typename I6 > + KOKKOS_FORCEINLINE_FUNCTION + size_type operator()( I0 const& i0, I1 const& i1, I2 const& i2 , I3 const& i3, I4 const& i4, I5 const& i5, I6 const& i6 ) const + { + return i6 + shape_type::N6 * ( + i5 + shape_type::N5 * ( + i4 + shape_type::N4 * ( + i3 + shape_type::N3 * ( + i2 + shape_type::N2 * ( i1 ))))) + + i0 * SR ; + } + + template < typename I0, typename I1, typename I2, typename I3 + ,typename I4, typename I5, typename I6, typename I7 > + KOKKOS_FORCEINLINE_FUNCTION + size_type operator()( I0 const& i0, I1 const& i1, I2 const& i2 , I3 const& i3, I4 const& i4, I5 const& i5, I6 const& i6, I7 const& i7 ) const + { + return i7 + shape_type::N7 * ( + i6 + shape_type::N6 * ( + i5 + shape_type::N5 * ( + i4 + shape_type::N4 * ( + i3 + shape_type::N3 * ( + i2 + shape_type::N2 * ( i1 )))))) + + i0 * SR ; + } +}; + +//---------------------------------------------------------------------------- +// LayoutStride : +template < class ShapeType > +struct ViewOffset< ShapeType , LayoutStride + , typename enable_if<( 0 < ShapeType::rank )>::type > + : public ShapeType +{ + typedef size_t size_type; + typedef ShapeType shape_type; + typedef LayoutStride array_layout ; + + size_type S[ shape_type::rank + 1 ]; + + template< class SType , class L > + KOKKOS_INLINE_FUNCTION + bool assign_subview( const ViewOffset & rhs + , const size_type n0 + , const size_type n1 + , const size_type n2 + , const size_type n3 + , const size_type n4 + , const size_type n5 + , const size_type n6 + , const size_type n7 + ) + { + shape_type::assign( *this, 0,0,0,0, 0,0,0,0 ); + + for ( int i = 0 ; i < int(shape_type::rank+1) ; ++i ) { S[i] = 0 ; } + + // preconditions: + // shape_type::rank <= rhs.rank + // shape_type::rank == count of nonzero( rhs_dim[i] ) + size_type dim[8] = { n0 , n1 , n2 , n3 , n4 , n5 , n6 , n7 }; + size_type str[ SType::rank + 1 ]; + + rhs.stride( str ); + + // contract the zero-dimensions + int r = 0 ; + for ( int i = 0 ; i < int(SType::rank) ; ++i ) { + if ( 0 != dim[i] ) { + dim[r] = dim[i] ; + str[r] = str[i] ; + ++r ; + } + } + + if ( int(shape_type::rank) == r ) { + // The shape is non-zero + for ( int i = 0 ; i < int(shape_type::rank) ; ++i ) { + const size_type cap = dim[i] * ( S[i] = str[i] ); + if ( S[ shape_type::rank ] < cap ) S[ shape_type::rank ] = cap ; + } + // set the contracted nonzero dimensions + shape_type::assign( *this, dim[0], dim[1], dim[2], dim[3], dim[4], dim[5], dim[6], dim[7] ); + } + + return true ; // definitely noncontiguous + } + + template< unsigned R > + KOKKOS_INLINE_FUNCTION + void assign( unsigned n ) + { assign_shape_dimension( *this , n ); } + + template< class ShapeRHS , class Layout > + KOKKOS_INLINE_FUNCTION + void assign( const ViewOffset & rhs + , typename enable_if<( int(ShapeRHS::rank) == int(shape_type::rank) )>::type * = 0 ) + { + rhs.stride(S); + shape_type::assign( *this, rhs.N0, rhs.N1, rhs.N2, rhs.N3, rhs.N4, rhs.N5, rhs.N6, rhs.N7 ); + } + + KOKKOS_INLINE_FUNCTION + void assign( const LayoutStride & layout ) + { + size_type max = 0 ; + for ( int i = 0 ; i < shape_type::rank ; ++i ) { + S[i] = layout.stride[i] ; + const size_type m = layout.dimension[i] * S[i] ; + if ( max < m ) { max = m ; } + } + S[ shape_type::rank ] = max ; + shape_type::assign( *this, layout.dimension[0], layout.dimension[1], + layout.dimension[2], layout.dimension[3], + layout.dimension[4], layout.dimension[5], + layout.dimension[6], layout.dimension[7] ); + } + + KOKKOS_INLINE_FUNCTION + void assign( size_t s0 , size_t s1 , size_t s2 , size_t s3 + , size_t s4 , size_t s5 , size_t s6 , size_t s7 + , size_t s8 ) + { + const size_t str[9] = { s0, s1, s2, s3, s4, s5, s6, s7, s8 }; + + // Last argument is the total length. + // Total length must be non-zero. + // All strides must be non-zero and less than total length. + bool ok = 0 < str[ shape_type::rank ] ; + + for ( int i = 0 ; ( i < shape_type::rank ) && + ( ok = 0 < str[i] && str[i] < str[ shape_type::rank ] ); ++i ); + + if ( ok ) { + size_t dim[8] = { 1,1,1,1,1,1,1,1 }; + int iorder[9] = { 0,0,0,0,0,0,0,0,0 }; + + // Ordering of strides smallest to largest. + for ( int i = 1 ; i < shape_type::rank ; ++i ) { + int j = i ; + for ( ; 0 < j && str[i] < str[ iorder[j-1] ] ; --j ) { + iorder[j] = iorder[j-1] ; + } + iorder[j] = i ; + } + + // Last argument is the total length. + iorder[ shape_type::rank ] = shape_type::rank ; + + // Determine dimension associated with each stride. + // Guarantees non-overlap by truncating dimension + // if ( 0 != str[ iorder[i+1] ] % str[ iorder[i] ] ) + for ( int i = 0 ; i < shape_type::rank ; ++i ) { + dim[ iorder[i] ] = str[ iorder[i+1] ] / str[ iorder[i] ] ; + } + + // Assign dimensions and strides: + shape_type::assign( *this, dim[0], dim[1], dim[2], dim[3], dim[4], dim[5], dim[6], dim[7] ); + for ( int i = 0 ; i <= shape_type::rank ; ++i ) { S[i] = str[i] ; } + } + else { + shape_type::assign(*this,0,0,0,0,0,0,0,0); + for ( int i = 0 ; i <= shape_type::rank ; ++i ) { S[i] = 0 ; } + } + } + + KOKKOS_INLINE_FUNCTION + void set_padding() {} + + KOKKOS_INLINE_FUNCTION + size_type cardinality() const + { return shape_type::N0 * shape_type::N1 * shape_type::N2 * shape_type::N3 * shape_type::N4 * shape_type::N5 * shape_type::N6 * shape_type::N7 ; } + + KOKKOS_INLINE_FUNCTION + size_type capacity() const { return S[ shape_type::rank ]; } + + template< typename iType > + KOKKOS_INLINE_FUNCTION + void stride( iType * const s ) const + { for ( int i = 0 ; i <= shape_type::rank ; ++i ) { s[i] = S[i] ; } } + + KOKKOS_INLINE_FUNCTION + size_type stride_0() const { return S[0] ; } + + KOKKOS_INLINE_FUNCTION + size_type stride_1() const { return S[1] ; } + + KOKKOS_INLINE_FUNCTION + size_type stride_2() const { return S[2] ; } + + KOKKOS_INLINE_FUNCTION + size_type stride_3() const { return S[3] ; } + + KOKKOS_INLINE_FUNCTION + size_type stride_4() const { return S[4] ; } + + KOKKOS_INLINE_FUNCTION + size_type stride_5() const { return S[5] ; } + + KOKKOS_INLINE_FUNCTION + size_type stride_6() const { return S[6] ; } + + KOKKOS_INLINE_FUNCTION + size_type stride_7() const { return S[7] ; } + + // rank 1 + template + KOKKOS_FORCEINLINE_FUNCTION + size_type operator()( I0 const& i0 ) const + { + return i0 * S[0] ; + } + + // rank 2 + template + KOKKOS_FORCEINLINE_FUNCTION + size_type operator()( I0 const& i0, I1 const& i1 ) const + { + return i0 * S[0] + i1 * S[1] ; + } + + template + KOKKOS_FORCEINLINE_FUNCTION + size_type operator()( I0 const& i0, I1 const& i1, I2 const& i2 ) const + { + return i0 * S[0] + i1 * S[1] + i2 * S[2] ; + } + + template + KOKKOS_FORCEINLINE_FUNCTION + size_type operator()( I0 const& i0, I1 const& i1, I2 const& i2 , I3 const& i3 ) const + { + return i0 * S[0] + i1 * S[1] + i2 * S[2] + i3 * S[3] ; + } + + template < typename I0, typename I1, typename I2, typename I3 + ,typename I4 > + KOKKOS_FORCEINLINE_FUNCTION + size_type operator()( I0 const& i0, I1 const& i1, I2 const& i2 , I3 const& i3, I4 const& i4 ) const + { + return i0 * S[0] + i1 * S[1] + i2 * S[2] + i3 * S[3] + i4 * S[4] ; + } + + template < typename I0, typename I1, typename I2, typename I3 + ,typename I4, typename I5 > + KOKKOS_FORCEINLINE_FUNCTION + size_type operator()( I0 const& i0, I1 const& i1, I2 const& i2 , I3 const& i3, I4 const& i4, I5 const& i5 ) const + { + return i0 * S[0] + i1 * S[1] + i2 * S[2] + i3 * S[3] + i4 * S[4] + i5 * S[5] ; + } + + template < typename I0, typename I1, typename I2, typename I3 + ,typename I4, typename I5, typename I6 > + KOKKOS_FORCEINLINE_FUNCTION + size_type operator()( I0 const& i0, I1 const& i1, I2 const& i2 , I3 const& i3, I4 const& i4, I5 const& i5, I6 const& i6 ) const + { + return i0 * S[0] + i1 * S[1] + i2 * S[2] + i3 * S[3] + i4 * S[4] + i5 * S[5] + i6 * S[6] ; + } + + template < typename I0, typename I1, typename I2, typename I3 + ,typename I4, typename I5, typename I6, typename I7 > + KOKKOS_FORCEINLINE_FUNCTION + size_type operator()( I0 const& i0, I1 const& i1, I2 const& i2 , I3 const& i3, I4 const& i4, I5 const& i5, I6 const& i6, I7 const& i7 ) const + { + return i0 * S[0] + i1 * S[1] + i2 * S[2] + i3 * S[3] + i4 * S[4] + i5 * S[5] + i6 * S[6] + i7 * S[7] ; + } +}; + +//---------------------------------------------------------------------------- + +template< class T > +struct ViewOffsetRange { + + enum { OK_integral_type = Impl::StaticAssert< Impl::is_integral::value >::value }; + + enum { is_range = false }; + + KOKKOS_INLINE_FUNCTION static + size_t dimension( size_t const , T const & ) { return 0 ; } + + KOKKOS_INLINE_FUNCTION static + size_t begin( T const & i ) { return size_t(i) ; } +}; + +template<> +struct ViewOffsetRange { + enum { is_range = false }; +}; + +template<> +struct ViewOffsetRange< Kokkos::ALL > { + enum { is_range = true }; + + KOKKOS_INLINE_FUNCTION static + size_t dimension( size_t const n , ALL const & ) { return n ; } + + KOKKOS_INLINE_FUNCTION static + size_t begin( ALL const & ) { return 0 ; } +}; + +template< typename iType > +struct ViewOffsetRange< std::pair > { + + enum { OK_integral_type = Impl::StaticAssert< Impl::is_integral::value >::value }; + + enum { is_range = true }; + + KOKKOS_INLINE_FUNCTION static + size_t dimension( size_t const n , std::pair const & r ) + { return ( size_t(r.first) < size_t(r.second) && size_t(r.second) <= n ) ? size_t(r.second) - size_t(r.first) : 0 ; } + + KOKKOS_INLINE_FUNCTION static + size_t begin( std::pair const & r ) { return size_t(r.first) ; } +}; + +template< typename iType > +struct ViewOffsetRange< Kokkos::pair > { + + enum { OK_integral_type = Impl::StaticAssert< Impl::is_integral::value >::value }; + + enum { is_range = true }; + + KOKKOS_INLINE_FUNCTION static + size_t dimension( size_t const n , Kokkos::pair const & r ) + { return ( size_t(r.first) < size_t(r.second) && size_t(r.second) <= n ) ? size_t(r.second) - size_t(r.first) : 0 ; } + + KOKKOS_INLINE_FUNCTION static + size_t begin( Kokkos::pair const & r ) { return size_t(r.first) ; } +}; + +}} // namespace Kokkos::Impl + +#endif //KOKKOS_VIEWOFFSET_HPP + diff --git a/lib/kokkos/core/src/impl/Kokkos_ViewSupport.hpp b/lib/kokkos/core/src/impl/Kokkos_ViewSupport.hpp new file mode 100755 index 0000000000..fbce4fb179 --- /dev/null +++ b/lib/kokkos/core/src/impl/Kokkos_ViewSupport.hpp @@ -0,0 +1,541 @@ +/* +//@HEADER +// ************************************************************************ +// +// Kokkos: Manycore Performance-Portable Multidimensional Arrays +// Copyright (2012) Sandia Corporation +// +// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +// the U.S. Government retains certain rights in this software. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// 1. Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// 3. Neither the name of the Corporation nor the names of the +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Questions? Contact H. Carter Edwards (hcedwar@sandia.gov) +// +// ************************************************************************ +//@HEADER +*/ + +#ifndef KOKKOS_VIEWSUPPORT_HPP +#define KOKKOS_VIEWSUPPORT_HPP + +#include +#include + +//---------------------------------------------------------------------------- +//---------------------------------------------------------------------------- + +namespace Kokkos { +namespace Impl { + +/** \brief Evaluate if LHS = RHS view assignment is allowed. */ +template< class ViewLHS , class ViewRHS > +struct ViewAssignable +{ + // Same memory space. + // Same value type. + // Compatible 'const' qualifier + // Cannot assign managed = unmannaged + enum { assignable_value = + ( is_same< typename ViewLHS::value_type , + typename ViewRHS::value_type >::value + || + is_same< typename ViewLHS::value_type , + typename ViewRHS::const_value_type >::value ) + && + is_same< typename ViewLHS::memory_space , + typename ViewRHS::memory_space >::value + && + ( ! ( ViewLHS::is_managed && ! ViewRHS::is_managed ) ) + }; + + enum { assignable_shape = + // Compatible shape and matching layout: + ( ShapeCompatible< typename ViewLHS::shape_type , + typename ViewRHS::shape_type >::value + && + is_same< typename ViewLHS::array_layout , + typename ViewRHS::array_layout >::value ) + || + // Matching layout, same rank, and LHS dynamic rank + ( is_same< typename ViewLHS::array_layout , + typename ViewRHS::array_layout >::value + && + int(ViewLHS::rank) == int(ViewRHS::rank) + && + int(ViewLHS::rank) == int(ViewLHS::rank_dynamic) ) + || + // Both rank-0, any shape and layout + ( int(ViewLHS::rank) == 0 && int(ViewRHS::rank) == 0 ) + || + // Both rank-1 and LHS is dynamic rank-1, any shape and layout + ( int(ViewLHS::rank) == 1 && int(ViewRHS::rank) == 1 && + int(ViewLHS::rank_dynamic) == 1 ) + }; + + enum { value = assignable_value && assignable_shape }; +}; + +} // namespace Impl +} // namespace Kokkos + +//---------------------------------------------------------------------------- +//---------------------------------------------------------------------------- + +namespace Kokkos { +namespace Impl { + +template< class ExecSpace , class Type , bool Initialize > +struct ViewDefaultConstruct +{ ViewDefaultConstruct( Type * , size_t ) {} }; + + +/** \brief ViewDataHandle provides the type of the 'data handle' which the view + * uses to access data with the [] operator. It also provides + * an allocate function and a function to extract a raw ptr from the + * data handle. ViewDataHandle also defines an enum ReferenceAble which + * specifies whether references/pointers to elements can be taken and a + * 'return_type' which is what the view operators will give back. + * Specialisation of this object allows three things depending + * on ViewTraits and compiler options: + * (i) Use special allocator (e.g. huge pages/small pages and pinned memory) + * (ii) Use special data handle type (e.g. add Cuda Texture Object) + * (iii) Use special access intrinsics (e.g. texture fetch and non-caching loads) + */ +template< class StaticViewTraits , class Enable = void > +struct ViewDataHandle { + + enum { ReturnTypeIsReference = true }; + + typedef typename StaticViewTraits::value_type * handle_type; + typedef typename StaticViewTraits::value_type & return_type; +}; + +template< class StaticViewTraits , class Enable = void > +class ViewDataManagement : public ViewDataHandle< StaticViewTraits > { +private: + + template< class , class > friend class ViewDataManagement ; + + struct PotentiallyManaged {}; + struct StaticallyUnmanaged {}; + + /* Statically unmanaged if traits or not executing in host-accessible memory space */ + typedef typename + Impl::if_c< StaticViewTraits::is_managed && + Impl::is_same< Kokkos::HostSpace + , Kokkos::Impl::ActiveExecutionMemorySpace >::value + , PotentiallyManaged + , StaticallyUnmanaged + >::type StaticManagementTag ; + + enum { Unmanaged = 0x01 + , Noncontiguous = 0x02 + }; + + enum { DefaultTraits = Impl::is_same< StaticManagementTag , StaticallyUnmanaged >::value ? Unmanaged : 0 }; + + unsigned m_traits ; ///< Runtime traits + + + template< class T > + inline static + unsigned assign( const ViewDataManagement & rhs , const PotentiallyManaged & ) + { return rhs.m_traits | ( rhs.is_managed() && Kokkos::HostSpace::in_parallel() ? unsigned(Unmanaged) : 0u ); } + + template< class T > + KOKKOS_INLINE_FUNCTION static + unsigned assign( const ViewDataManagement & rhs , const StaticallyUnmanaged & ) + { return rhs.m_traits | Unmanaged ; } + + inline + void increment( const void * ptr , const PotentiallyManaged & ) const + { if ( is_managed() ) StaticViewTraits::memory_space::increment( ptr ); } + + inline + void decrement( const void * ptr , const PotentiallyManaged & ) const + { if ( is_managed() ) StaticViewTraits::memory_space::decrement( ptr ); } + + KOKKOS_INLINE_FUNCTION + void increment( const void * , const StaticallyUnmanaged & ) const {} + + KOKKOS_INLINE_FUNCTION + void decrement( const void * , const StaticallyUnmanaged & ) const {} + +public: + + typedef typename ViewDataHandle< StaticViewTraits >::handle_type handle_type; + + KOKKOS_INLINE_FUNCTION + ViewDataManagement() : m_traits( DefaultTraits ) {} + + KOKKOS_INLINE_FUNCTION + ViewDataManagement( const ViewDataManagement & rhs ) + : m_traits( assign( rhs , StaticManagementTag() ) ) {} + + KOKKOS_INLINE_FUNCTION + ViewDataManagement & operator = ( const ViewDataManagement & rhs ) + { m_traits = assign( rhs , StaticManagementTag() ); return *this ; } + + template< class SVT > + KOKKOS_INLINE_FUNCTION + ViewDataManagement( const ViewDataManagement & rhs ) + : m_traits( assign( rhs , StaticManagementTag() ) ) {} + + template< class SVT > + KOKKOS_INLINE_FUNCTION + ViewDataManagement & operator = ( const ViewDataManagement & rhs ) + { m_traits = assign( rhs , StaticManagementTag() ); return *this ; } + + KOKKOS_INLINE_FUNCTION + bool is_managed() const { return ! ( m_traits & Unmanaged ); } + + KOKKOS_INLINE_FUNCTION + bool is_contiguous() const { return ! ( m_traits & Noncontiguous ); } + + KOKKOS_INLINE_FUNCTION + void set_unmanaged() { m_traits |= Unmanaged ; } + + KOKKOS_INLINE_FUNCTION + void set_noncontiguous() { m_traits |= Noncontiguous ; } + + + KOKKOS_INLINE_FUNCTION + void increment( handle_type handle ) const + { increment( ( typename StaticViewTraits::value_type *) handle , StaticManagementTag() ); } + + KOKKOS_INLINE_FUNCTION + void decrement( handle_type handle ) const + { decrement( ( typename StaticViewTraits::value_type *) handle , StaticManagementTag() ); } + + + KOKKOS_INLINE_FUNCTION + void increment( const void * ptr ) const + { increment( ptr , StaticManagementTag() ); } + + KOKKOS_INLINE_FUNCTION + void decrement( const void * ptr ) const + { decrement( ptr , StaticManagementTag() ); } + + + template< bool Initialize > + static + handle_type allocate( const std::string & label + , const Impl::ViewOffset< typename StaticViewTraits::shape_type + , typename StaticViewTraits::array_layout > & offset_map ) + { + typedef typename StaticViewTraits::execution_space execution_space ; + typedef typename StaticViewTraits::memory_space memory_space ; + typedef typename StaticViewTraits::value_type value_type ; + + const size_t count = offset_map.capacity(); + + value_type * ptr = (value_type*) memory_space::allocate( label , sizeof(value_type) * count ); + + // Default construct within the view's execution space. + (void) ViewDefaultConstruct< execution_space , value_type , Initialize >( ptr , count ); + + return typename ViewDataHandle< StaticViewTraits >::handle_type(ptr); + } +}; + +} // namespace Impl +} // namespace Kokkos + +//---------------------------------------------------------------------------- +//---------------------------------------------------------------------------- + +namespace Kokkos { +namespace Impl { + +template< class OutputView , class InputView , unsigned Rank = OutputView::Rank > +struct ViewRemap +{ + typedef typename OutputView::size_type size_type ; + + const OutputView output ; + const InputView input ; + const size_type n0 ; + const size_type n1 ; + const size_type n2 ; + const size_type n3 ; + const size_type n4 ; + const size_type n5 ; + const size_type n6 ; + const size_type n7 ; + + ViewRemap( const OutputView & arg_out , const InputView & arg_in ) + : output( arg_out ), input( arg_in ) + , n0( std::min( (size_t)arg_out.dimension_0() , (size_t)arg_in.dimension_0() ) ) + , n1( std::min( (size_t)arg_out.dimension_1() , (size_t)arg_in.dimension_1() ) ) + , n2( std::min( (size_t)arg_out.dimension_2() , (size_t)arg_in.dimension_2() ) ) + , n3( std::min( (size_t)arg_out.dimension_3() , (size_t)arg_in.dimension_3() ) ) + , n4( std::min( (size_t)arg_out.dimension_4() , (size_t)arg_in.dimension_4() ) ) + , n5( std::min( (size_t)arg_out.dimension_5() , (size_t)arg_in.dimension_5() ) ) + , n6( std::min( (size_t)arg_out.dimension_6() , (size_t)arg_in.dimension_6() ) ) + , n7( std::min( (size_t)arg_out.dimension_7() , (size_t)arg_in.dimension_7() ) ) + { + typedef typename OutputView::execution_space execution_space ; + Kokkos::RangePolicy< execution_space > range( 0 , n0 ); + parallel_for( range , *this ); + } + + KOKKOS_INLINE_FUNCTION + void operator()( const size_type i0 ) const + { + for ( size_type i1 = 0 ; i1 < n1 ; ++i1 ) { + for ( size_type i2 = 0 ; i2 < n2 ; ++i2 ) { + for ( size_type i3 = 0 ; i3 < n3 ; ++i3 ) { + for ( size_type i4 = 0 ; i4 < n4 ; ++i4 ) { + for ( size_type i5 = 0 ; i5 < n5 ; ++i5 ) { + for ( size_type i6 = 0 ; i6 < n6 ; ++i6 ) { + for ( size_type i7 = 0 ; i7 < n7 ; ++i7 ) { + output.at(i0,i1,i2,i3,i4,i5,i6,i7) = input.at(i0,i1,i2,i3,i4,i5,i6,i7); + }}}}}}} + } +}; + +template< class OutputView , class InputView > +struct ViewRemap< OutputView , InputView , 0 > +{ + typedef typename OutputView::value_type value_type ; + typedef typename OutputView::memory_space dst_space ; + typedef typename InputView ::memory_space src_space ; + + ViewRemap( const OutputView & arg_out , const InputView & arg_in ) + { + DeepCopy< dst_space , src_space >( arg_out.ptr_on_device() , + arg_in.ptr_on_device() , + sizeof(value_type) ); + } +}; + +//---------------------------------------------------------------------------- + +template< class ExecSpace , class Type > +struct ViewDefaultConstruct< ExecSpace , Type , true > +{ + Type * const m_ptr ; + + KOKKOS_INLINE_FUNCTION + void operator()( const typename ExecSpace::size_type i ) const + { new( m_ptr + i ) Type(); } + + ViewDefaultConstruct( Type * pointer , size_t capacity ) + : m_ptr( pointer ) + { + Kokkos::RangePolicy< ExecSpace > range( 0 , capacity ); + parallel_for( range , *this ); + ExecSpace::fence(); + } +}; + +template< class OutputView , unsigned Rank = OutputView::Rank > +struct ViewFill +{ + typedef typename OutputView::const_value_type const_value_type ; + typedef typename OutputView::size_type size_type ; + + const OutputView output ; + const_value_type input ; + + ViewFill( const OutputView & arg_out , const_value_type & arg_in ) + : output( arg_out ), input( arg_in ) + { + typedef typename OutputView::execution_space execution_space ; + Kokkos::RangePolicy< execution_space > range( 0 , output.dimension_0() ); + parallel_for( range , *this ); + execution_space::fence(); + } + + KOKKOS_INLINE_FUNCTION + void operator()( const size_type i0 ) const + { + for ( size_type i1 = 0 ; i1 < output.dimension_1() ; ++i1 ) { + for ( size_type i2 = 0 ; i2 < output.dimension_2() ; ++i2 ) { + for ( size_type i3 = 0 ; i3 < output.dimension_3() ; ++i3 ) { + for ( size_type i4 = 0 ; i4 < output.dimension_4() ; ++i4 ) { + for ( size_type i5 = 0 ; i5 < output.dimension_5() ; ++i5 ) { + for ( size_type i6 = 0 ; i6 < output.dimension_6() ; ++i6 ) { + for ( size_type i7 = 0 ; i7 < output.dimension_7() ; ++i7 ) { + output.at(i0,i1,i2,i3,i4,i5,i6,i7) = input ; + }}}}}}} + } +}; + +template< class OutputView > +struct ViewFill< OutputView , 0 > +{ + typedef typename OutputView::const_value_type const_value_type ; + typedef typename OutputView::memory_space dst_space ; + + ViewFill( const OutputView & arg_out , const_value_type & arg_in ) + { + DeepCopy< dst_space , dst_space >( arg_out.ptr_on_device() , & arg_in , + sizeof(const_value_type) ); + } +}; + +} // namespace Impl +} // namespace Kokkos + +//---------------------------------------------------------------------------- +//---------------------------------------------------------------------------- + +namespace Kokkos { + +struct ViewAllocateWithoutInitializing { + + const std::string label ; + + ViewAllocateWithoutInitializing() : label() {} + ViewAllocateWithoutInitializing( const std::string & arg_label ) : label( arg_label ) {} + ViewAllocateWithoutInitializing( const char * const arg_label ) : label( arg_label ) {} +}; + +struct ViewAllocate { + + const std::string label ; + + ViewAllocate() : label() {} + ViewAllocate( const std::string & arg_label ) : label( arg_label ) {} + ViewAllocate( const char * const arg_label ) : label( arg_label ) {} +}; + +} + +namespace Kokkos { +namespace Impl { + +template< class Traits , class AllocationProperties , class Enable = void > +struct ViewAllocProp : public Kokkos::Impl::false_type {}; + +template< class Traits > +struct ViewAllocProp< Traits , Kokkos::ViewAllocate + , typename Kokkos::Impl::enable_if<( + Traits::is_managed && ! Kokkos::Impl::is_const< typename Traits::value_type >::value + )>::type > + : public Kokkos::Impl::true_type +{ + typedef size_t size_type ; + typedef const ViewAllocate & property_type ; + + enum { Initialize = true }; + enum { AllowPadding = false }; + + inline + static const std::string & label( property_type p ) { return p.label ; } +}; + +template< class Traits > +struct ViewAllocProp< Traits , std::string + , typename Kokkos::Impl::enable_if<( + Traits::is_managed && ! Kokkos::Impl::is_const< typename Traits::value_type >::value + )>::type > + : public Kokkos::Impl::true_type +{ + typedef size_t size_type ; + typedef const std::string & property_type ; + + enum { Initialize = true }; + enum { AllowPadding = false }; + + inline + static const std::string & label( property_type s ) { return s ; } +}; + +template< class Traits , unsigned N > +struct ViewAllocProp< Traits , char[N] + , typename Kokkos::Impl::enable_if<( + Traits::is_managed && ! Kokkos::Impl::is_const< typename Traits::value_type >::value + )>::type > + : public Kokkos::Impl::true_type +{ +private: + typedef char label_type[N] ; +public: + + typedef size_t size_type ; + typedef const label_type & property_type ; + + enum { Initialize = true }; + enum { AllowPadding = false }; + + inline + static std::string label( property_type s ) { return std::string(s) ; } +}; + +template< class Traits > +struct ViewAllocProp< Traits , Kokkos::ViewAllocateWithoutInitializing + , typename Kokkos::Impl::enable_if<( + Traits::is_managed && ! Kokkos::Impl::is_const< typename Traits::value_type >::value + )>::type > + : public Kokkos::Impl::true_type +{ + typedef size_t size_type ; + typedef const Kokkos::ViewAllocateWithoutInitializing & property_type ; + + enum { Initialize = false }; + enum { AllowPadding = false }; + + inline + static std::string label( property_type s ) { return s.label ; } +}; + +} // namespace Impl +} // namespace Kokkos + +//---------------------------------------------------------------------------- +//---------------------------------------------------------------------------- + +namespace Kokkos { +namespace Impl { + +template< class Traits , class PointerProperties , class Enable = void > +struct ViewRawPointerProp : public Kokkos::Impl::false_type {}; + +template< class Traits , typename T > +struct ViewRawPointerProp< Traits , T , + typename Kokkos::Impl::enable_if<( + Impl::is_same< T , typename Traits::value_type >::value || + Impl::is_same< T , typename Traits::non_const_value_type >::value + )>::type > + : public Kokkos::Impl::true_type +{ + typedef size_t size_type ; +}; + +} // namespace Impl +} // namespace Kokkos + +//---------------------------------------------------------------------------- +//---------------------------------------------------------------------------- + +#endif /* #ifndef KOKKOS_VIEWSUPPORT_HPP */ + + diff --git a/lib/kokkos/core/src/impl/Kokkos_ViewTileLeft.hpp b/lib/kokkos/core/src/impl/Kokkos_ViewTileLeft.hpp new file mode 100755 index 0000000000..7a9afc4ee4 --- /dev/null +++ b/lib/kokkos/core/src/impl/Kokkos_ViewTileLeft.hpp @@ -0,0 +1,195 @@ +/* +//@HEADER +// ************************************************************************ +// +// Kokkos: Manycore Performance-Portable Multidimensional Arrays +// Copyright (2012) Sandia Corporation +// +// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +// the U.S. Government retains certain rights in this software. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// 1. Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// 3. Neither the name of the Corporation nor the names of the +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Questions? Contact H. Carter Edwards (hcedwar@sandia.gov) +// +// ************************************************************************ +//@HEADER +*/ + +#ifndef KOKKOS_VIEWTILELEFT_HPP +#define KOKKOS_VIEWTILELEFT_HPP + +//---------------------------------------------------------------------------- +//---------------------------------------------------------------------------- + +namespace Kokkos { +namespace Impl { + +template< class T , unsigned N0 , unsigned N1 , class MemorySpace , class MemoryTraits > +struct ViewSpecialize< T , void , LayoutTileLeft , MemorySpace , MemoryTraits > +{ + typedef ViewDefault type ; +}; + +struct ViewTile {}; + +template< class ShapeType , unsigned N0 , unsigned N1 > +struct ViewOffset< ShapeType + , LayoutTileLeft /* Only accept properly shaped tiles */ + , typename Impl::enable_if<( 2 == ShapeType::rank + && + 2 == ShapeType::rank_dynamic + )>::type > + : public ShapeType +{ + enum { SHIFT_0 = Impl::power_of_two::value }; + enum { SHIFT_1 = Impl::power_of_two::value }; + enum { MASK_0 = N0 - 1 }; + enum { MASK_1 = N1 - 1 }; + + typedef size_t size_type ; + typedef ShapeType shape_type ; + typedef LayoutTileLeft array_layout ; + + enum { has_padding = true }; + + size_type tile_N0 ; + + KOKKOS_INLINE_FUNCTION + void assign( const ViewOffset & rhs ) + { + shape_type::N0 = rhs.N0 ; + shape_type::N1 = rhs.N1 ; + tile_N0 = ( rhs.N0 + MASK_0 ) >> SHIFT_0 ; // number of tiles in first dimension + } + + KOKKOS_INLINE_FUNCTION + void assign( size_t n0 , size_t n1 + , int = 0 , int = 0 + , int = 0 , int = 0 + , int = 0 , int = 0 + , int = 0 + ) + { + shape_type::N0 = n0 ; + shape_type::N1 = n1 ; + tile_N0 = ( n0 + MASK_0 ) >> SHIFT_0 ; // number of tiles in first dimension + } + + + KOKKOS_INLINE_FUNCTION + void set_padding() {} + + + template< typename I0 , typename I1 > + KOKKOS_INLINE_FUNCTION + size_type operator()( I0 const & i0 , I1 const & i1 + , int = 0 , int = 0 + , int = 0 , int = 0 + , int = 0 , int = 0 + ) const + { + return /* ( ( Tile offset ) * ( Tile size ) ) */ + ( ( (i0>>SHIFT_0) + tile_N0 * (i1>>SHIFT_1) ) << (SHIFT_0 + SHIFT_1) ) + + /* ( Offset within tile ) */ + ( (i0 & MASK_0) + ((i1 & MASK_1)< + KOKKOS_INLINE_FUNCTION + size_type tile_begin( I0 const & i_tile0 , I1 const & i_tile1 ) const + { + return ( i_tile0 + tile_N0 * i_tile1 ) << ( SHIFT_0 + SHIFT_1 ); + } + + + KOKKOS_INLINE_FUNCTION + size_type capacity() const + { + // ( TileDim0 * ( TileDim1 ) ) * TileSize + return ( tile_N0 * ( ( shape_type::N1 + MASK_1 ) >> SHIFT_1 ) ) << ( SHIFT_0 + SHIFT_1 ); + } +}; + +template<> +struct ViewAssignment< ViewTile , void , void > +{ + // Some compilers have type-matching issues on the integer values when using: + // template< class T , unsigned N0 , unsigned N1 , class A2 , class A3 > + template< class T , unsigned dN0 , unsigned dN1 + , class A2 , class A3 + , unsigned sN0 , unsigned sN1 > + KOKKOS_INLINE_FUNCTION + ViewAssignment( View< T[dN0][dN1], LayoutLeft, A2, A3, Impl::ViewDefault > & dst + , View< T** , LayoutTileLeft, A2, A3, Impl::ViewDefault > const & src + , size_t const i_tile0 + , typename Impl::enable_if< unsigned(dN0) == unsigned(sN0) && + unsigned(dN1) == unsigned(sN1) + , size_t const + >::type i_tile1 + ) + { + // Destination is always contiguous but source may be non-contiguous + // so don't assign the whole view management object. + // Just query and appropriately set the reference-count state. + + if ( ! src.m_management.is_managed() ) dst.m_management.set_unmanaged(); + + dst.m_ptr_on_device = src.m_ptr_on_device + src.m_offset_map.tile_begin(i_tile0,i_tile1); + + dst.m_management.increment( dst.m_ptr_on_device ); + } +}; + +} /* namespace Impl */ +} /* namespace Kokkos */ + +namespace Kokkos { + +template< class T , unsigned N0, unsigned N1, class A2, class A3 > +KOKKOS_INLINE_FUNCTION +View< T[N0][N1], LayoutLeft, A2, A3, Impl::ViewDefault > +tile_subview( const View,A2,A3,Impl::ViewDefault> & src + , const size_t i_tile0 + , const size_t i_tile1 + ) +{ + View< T[N0][N1], LayoutLeft, A2, A3, Impl::ViewDefault > dst ; + + (void) Impl::ViewAssignment< Impl::ViewTile , void , void >( dst , src , i_tile0 , i_tile1 ); + + return dst ; +} + +} /* namespace Kokkos */ + +//---------------------------------------------------------------------------- +//---------------------------------------------------------------------------- + +#endif /* #ifndef KOKKOS_VIEWTILELEFT_HPP */ + diff --git a/lib/kokkos/core/src/impl/Kokkos_Volatile_Load.hpp b/lib/kokkos/core/src/impl/Kokkos_Volatile_Load.hpp new file mode 100755 index 0000000000..ea349e7ab1 --- /dev/null +++ b/lib/kokkos/core/src/impl/Kokkos_Volatile_Load.hpp @@ -0,0 +1,242 @@ +/* +//@HEADER +// ************************************************************************ +// +// Kokkos: Manycore Performance-Portable Multidimensional Arrays +// Copyright (2012) Sandia Corporation +// +// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +// the U.S. Government retains certain rights in this software. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// 1. Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// 3. Neither the name of the Corporation nor the names of the +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Questions? Contact H. Carter Edwards (hcedwar@sandia.gov) +// +// ************************************************************************ +//@HEADER +*/ + +#if defined( KOKKOS_ATOMIC_HPP ) && ! defined( KOKKOS_VOLATILE_LOAD ) +#define KOKKOS_VOLATILE_LOAD + +#if defined( __GNUC__ ) /* GNU C */ || \ + defined( __GNUG__ ) /* GNU C++ */ || \ + defined( __clang__ ) + +#define KOKKOS_MAY_ALIAS __attribute__((__may_alias__)) + +#else + +#define KOKKOS_MAY_ALIAS + +#endif + +namespace Kokkos { + +//---------------------------------------------------------------------------- + +template +KOKKOS_FORCEINLINE_FUNCTION +T volatile_load(T const volatile * const src_ptr) +{ + typedef uint64_t KOKKOS_MAY_ALIAS T64; + typedef uint32_t KOKKOS_MAY_ALIAS T32; + typedef uint16_t KOKKOS_MAY_ALIAS T16; + typedef uint8_t KOKKOS_MAY_ALIAS T8; + + enum { + NUM_8 = sizeof(T), + NUM_16 = NUM_8 / 2, + NUM_32 = NUM_8 / 4, + NUM_64 = NUM_8 / 8 + }; + + union { + T const volatile * const ptr; + T64 const volatile * const ptr64; + T32 const volatile * const ptr32; + T16 const volatile * const ptr16; + T8 const volatile * const ptr8; + } src = {src_ptr}; + + T result; + + union { + T * const ptr; + T64 * const ptr64; + T32 * const ptr32; + T16 * const ptr16; + T8 * const ptr8; + } dst = {&result}; + + for (int i=0; i < NUM_64; ++i) { + dst.ptr64[i] = src.ptr64[i]; + } + + if ( NUM_64*2 < NUM_32 ) { + dst.ptr32[NUM_64*2] = src.ptr32[NUM_64*2]; + } + + if ( NUM_32*2 < NUM_16 ) { + dst.ptr16[NUM_32*2] = src.ptr16[NUM_32*2]; + } + + if ( NUM_16*2 < NUM_8 ) { + dst.ptr8[NUM_16*2] = src.ptr8[NUM_16*2]; + } + + return result; +} + +template +KOKKOS_FORCEINLINE_FUNCTION +void volatile_store(T volatile * const dst_ptr, T const volatile * const src_ptr) +{ + typedef uint64_t KOKKOS_MAY_ALIAS T64; + typedef uint32_t KOKKOS_MAY_ALIAS T32; + typedef uint16_t KOKKOS_MAY_ALIAS T16; + typedef uint8_t KOKKOS_MAY_ALIAS T8; + + enum { + NUM_8 = sizeof(T), + NUM_16 = NUM_8 / 2, + NUM_32 = NUM_8 / 4, + NUM_64 = NUM_8 / 8 + }; + + union { + T const volatile * const ptr; + T64 const volatile * const ptr64; + T32 const volatile * const ptr32; + T16 const volatile * const ptr16; + T8 const volatile * const ptr8; + } src = {src_ptr}; + + union { + T volatile * const ptr; + T64 volatile * const ptr64; + T32 volatile * const ptr32; + T16 volatile * const ptr16; + T8 volatile * const ptr8; + } dst = {dst_ptr}; + + for (int i=0; i < NUM_64; ++i) { + dst.ptr64[i] = src.ptr64[i]; + } + + if ( NUM_64*2 < NUM_32 ) { + dst.ptr32[NUM_64*2] = src.ptr32[NUM_64*2]; + } + + if ( NUM_32*2 < NUM_16 ) { + dst.ptr16[NUM_32*2] = src.ptr16[NUM_32*2]; + } + + if ( NUM_16*2 < NUM_8 ) { + dst.ptr8[NUM_16*2] = src.ptr8[NUM_16*2]; + } +} + +template +KOKKOS_FORCEINLINE_FUNCTION +void volatile_store(T volatile * const dst_ptr, T const * const src_ptr) +{ + typedef uint64_t KOKKOS_MAY_ALIAS T64; + typedef uint32_t KOKKOS_MAY_ALIAS T32; + typedef uint16_t KOKKOS_MAY_ALIAS T16; + typedef uint8_t KOKKOS_MAY_ALIAS T8; + + enum { + NUM_8 = sizeof(T), + NUM_16 = NUM_8 / 2, + NUM_32 = NUM_8 / 4, + NUM_64 = NUM_8 / 8 + }; + + union { + T const * const ptr; + T64 const * const ptr64; + T32 const * const ptr32; + T16 const * const ptr16; + T8 const * const ptr8; + } src = {src_ptr}; + + union { + T volatile * const ptr; + T64 volatile * const ptr64; + T32 volatile * const ptr32; + T16 volatile * const ptr16; + T8 volatile * const ptr8; + } dst = {dst_ptr}; + + for (int i=0; i < NUM_64; ++i) { + dst.ptr64[i] = src.ptr64[i]; + } + + if ( NUM_64*2 < NUM_32 ) { + dst.ptr32[NUM_64*2] = src.ptr32[NUM_64*2]; + } + + if ( NUM_32*2 < NUM_16 ) { + dst.ptr16[NUM_32*2] = src.ptr16[NUM_32*2]; + } + + if ( NUM_16*2 < NUM_8 ) { + dst.ptr8[NUM_16*2] = src.ptr8[NUM_16*2]; + } +} + +template +KOKKOS_FORCEINLINE_FUNCTION +void volatile_store(T volatile * dst_ptr, T const volatile & src) +{ volatile_store(dst_ptr, &src); } + +template +KOKKOS_FORCEINLINE_FUNCTION +void volatile_store(T volatile * dst_ptr, T const & src) +{ volatile_store(dst_ptr, &src); } + +template +KOKKOS_FORCEINLINE_FUNCTION +T safe_load(T const * const ptr) +{ +#if !defined( __MIC__ ) + return *ptr; +#else + return volatile_load(ptr); +#endif +} + +} // namespace kokkos + +#undef KOKKOS_MAY_ALIAS + +#endif + + + diff --git a/lib/kokkos/core/src/impl/Kokkos_hwloc.cpp b/lib/kokkos/core/src/impl/Kokkos_hwloc.cpp new file mode 100755 index 0000000000..f192d4716c --- /dev/null +++ b/lib/kokkos/core/src/impl/Kokkos_hwloc.cpp @@ -0,0 +1,704 @@ +/* +//@HEADER +// ************************************************************************ +// +// Kokkos: Manycore Performance-Portable Multidimensional Arrays +// Copyright (2012) Sandia Corporation +// +// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +// the U.S. Government retains certain rights in this software. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// 1. Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// 3. Neither the name of the Corporation nor the names of the +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Questions? Contact H. Carter Edwards (hcedwar@sandia.gov) +// +// ************************************************************************ +//@HEADER +*/ + +#define DEBUG_PRINT 0 + +#include +#include + +#include +#include +#include + +/*--------------------------------------------------------------------------*/ +/*--------------------------------------------------------------------------*/ + +namespace Kokkos { +namespace hwloc { + +/* Return 0 if asynchronous, 1 if synchronous and include process. */ +unsigned thread_mapping( const char * const label , + const bool allow_async , + unsigned & thread_count , + unsigned & use_numa_count , + unsigned & use_cores_per_numa , + std::pair threads_coord[] ) +{ + const bool hwloc_avail = Kokkos::hwloc::available(); + const unsigned avail_numa_count = hwloc_avail ? hwloc::get_available_numa_count() : 1 ; + const unsigned avail_cores_per_numa = hwloc_avail ? hwloc::get_available_cores_per_numa() : thread_count ; + const unsigned avail_threads_per_core = hwloc_avail ? hwloc::get_available_threads_per_core() : 1 ; + + // (numa,core) coordinate of the process: + const std::pair proc_coord = Kokkos::hwloc::get_this_thread_coordinate(); + + //------------------------------------------------------------------------ + // Defaults for unspecified inputs: + + if ( ! use_numa_count ) { + // Default to use all NUMA regions + use_numa_count = ! thread_count ? avail_numa_count : ( + thread_count < avail_numa_count ? thread_count : avail_numa_count ); + } + + if ( ! use_cores_per_numa ) { + // Default to use all but one core if asynchronous, all cores if synchronous. + const unsigned threads_per_numa = thread_count / use_numa_count ; + + use_cores_per_numa = ! threads_per_numa ? avail_cores_per_numa - ( allow_async ? 1 : 0 ) : ( + threads_per_numa < avail_cores_per_numa ? threads_per_numa : avail_cores_per_numa ); + } + + if ( ! thread_count ) { + thread_count = use_numa_count * use_cores_per_numa * avail_threads_per_core ; + } + + //------------------------------------------------------------------------ + // Input verification: + + const bool valid_numa = use_numa_count <= avail_numa_count ; + const bool valid_cores = use_cores_per_numa && + use_cores_per_numa <= avail_cores_per_numa ; + const bool valid_threads = thread_count && + thread_count <= use_numa_count * use_cores_per_numa * avail_threads_per_core ; + const bool balanced_numa = ! ( thread_count % use_numa_count ); + const bool balanced_cores = ! ( thread_count % ( use_numa_count * use_cores_per_numa ) ); + + const bool valid_input = valid_numa && valid_cores && valid_threads && balanced_numa && balanced_cores ; + + if ( ! valid_input ) { + + std::ostringstream msg ; + + msg << label << " HWLOC ERROR(s)" ; + + if ( ! valid_threads ) { + msg << " : thread_count(" << thread_count + << ") exceeds capacity(" + << use_numa_count * use_cores_per_numa * avail_threads_per_core + << ")" ; + } + if ( ! valid_numa ) { + msg << " : use_numa_count(" << use_numa_count + << ") exceeds capacity(" << avail_numa_count << ")" ; + } + if ( ! valid_cores ) { + msg << " : use_cores_per_numa(" << use_cores_per_numa + << ") exceeds capacity(" << avail_cores_per_numa << ")" ; + } + if ( ! balanced_numa ) { + msg << " : thread_count(" << thread_count + << ") imbalanced among numa(" << use_numa_count << ")" ; + } + if ( ! balanced_cores ) { + msg << " : thread_count(" << thread_count + << ") imbalanced among cores(" << use_numa_count * use_cores_per_numa << ")" ; + } + + Kokkos::Impl::throw_runtime_exception( msg.str() ); + } + + const unsigned thread_spawn_synchronous = + ( allow_async && + 1 < thread_count && + ( use_numa_count < avail_numa_count || + use_cores_per_numa < avail_cores_per_numa ) ) + ? 0 /* asyncronous */ + : 1 /* synchronous, threads_coord[0] is process core */ ; + + // Determine binding coordinates for to-be-spawned threads so that + // threads may be bound to cores as they are spawned. + + const unsigned threads_per_core = thread_count / ( use_numa_count * use_cores_per_numa ); + + if ( thread_spawn_synchronous ) { + // Working synchronously and include process core as threads_coord[0]. + // Swap the NUMA coordinate of the process core with 0 + // Swap the CORE coordinate of the process core with 0 + for ( unsigned i = 0 , inuma = avail_numa_count - use_numa_count ; inuma < avail_numa_count ; ++inuma ) { + const unsigned numa_coord = 0 == inuma ? proc_coord.first : ( proc_coord.first == inuma ? 0 : inuma ); + for ( unsigned icore = avail_cores_per_numa - use_cores_per_numa ; icore < avail_cores_per_numa ; ++icore ) { + const unsigned core_coord = 0 == icore ? proc_coord.second : ( proc_coord.second == icore ? 0 : icore ); + for ( unsigned ith = 0 ; ith < threads_per_core ; ++ith , ++i ) { + threads_coord[i].first = numa_coord ; + threads_coord[i].second = core_coord ; + } + } + } + } + else if ( use_numa_count < avail_numa_count ) { + // Working asynchronously and omit the process' NUMA region from the pool. + // Swap the NUMA coordinate of the process core with ( ( avail_numa_count - use_numa_count ) - 1 ) + const unsigned numa_coord_swap = ( avail_numa_count - use_numa_count ) - 1 ; + for ( unsigned i = 0 , inuma = avail_numa_count - use_numa_count ; inuma < avail_numa_count ; ++inuma ) { + const unsigned numa_coord = proc_coord.first == inuma ? numa_coord_swap : inuma ; + for ( unsigned icore = avail_cores_per_numa - use_cores_per_numa ; icore < avail_cores_per_numa ; ++icore ) { + const unsigned core_coord = icore ; + for ( unsigned ith = 0 ; ith < threads_per_core ; ++ith , ++i ) { + threads_coord[i].first = numa_coord ; + threads_coord[i].second = core_coord ; + } + } + } + } + else if ( use_cores_per_numa < avail_cores_per_numa ) { + // Working asynchronously and omit the process' core from the pool. + // Swap the CORE coordinate of the process core with ( ( avail_cores_per_numa - use_cores_per_numa ) - 1 ) + const unsigned core_coord_swap = ( avail_cores_per_numa - use_cores_per_numa ) - 1 ; + for ( unsigned i = 0 , inuma = avail_numa_count - use_numa_count ; inuma < avail_numa_count ; ++inuma ) { + const unsigned numa_coord = inuma ; + for ( unsigned icore = avail_cores_per_numa - use_cores_per_numa ; icore < avail_cores_per_numa ; ++icore ) { + const unsigned core_coord = proc_coord.second == icore ? core_coord_swap : icore ; + for ( unsigned ith = 0 ; ith < threads_per_core ; ++ith , ++i ) { + threads_coord[i].first = numa_coord ; + threads_coord[i].second = core_coord ; + } + } + } + } + + return thread_spawn_synchronous ; +} + +} /* namespace hwloc */ +} /* namespace Kokkos */ + +/*--------------------------------------------------------------------------*/ +/*--------------------------------------------------------------------------*/ + +#if defined( KOKKOS_HAVE_HWLOC ) + +#include +#include +#include + +/*--------------------------------------------------------------------------*/ +/* Third Party Libraries */ + +/* Hardware locality library: http://www.open-mpi.org/projects/hwloc/ */ +#include + +#define REQUIRED_HWLOC_API_VERSION 0x000010300 + +#if HWLOC_API_VERSION < REQUIRED_HWLOC_API_VERSION +#error "Requires http://www.open-mpi.org/projects/hwloc/ Version 1.3 or greater" +#endif + +/*--------------------------------------------------------------------------*/ + +namespace Kokkos { +namespace hwloc { +namespace { + +#if DEBUG_PRINT + +inline +void print_bitmap( std::ostream & s , const hwloc_const_bitmap_t bitmap ) +{ + s << "{" ; + for ( int i = hwloc_bitmap_first( bitmap ) ; + -1 != i ; i = hwloc_bitmap_next( bitmap , i ) ) { + s << " " << i ; + } + s << " }" ; +} + +#endif + +enum { MAX_CORE = 1024 }; + +std::pair s_core_topology(0,0); +unsigned s_core_capacity(0); +hwloc_topology_t s_hwloc_topology(0); +hwloc_bitmap_t s_hwloc_location(0); +hwloc_bitmap_t s_process_binding(0); +hwloc_bitmap_t s_core[ MAX_CORE ]; + +struct Sentinel { + ~Sentinel(); + Sentinel(); +}; + +bool sentinel() +{ + static Sentinel self ; + + if ( 0 == s_hwloc_topology ) { + std::cerr << "Kokkos::hwloc ERROR : Called after return from main()" << std::endl ; + std::cerr.flush(); + } + + return 0 != s_hwloc_topology ; +} + +Sentinel::~Sentinel() +{ + hwloc_topology_destroy( s_hwloc_topology ); + hwloc_bitmap_free( s_process_binding ); + hwloc_bitmap_free( s_hwloc_location ); + + s_core_topology.first = 0 ; + s_core_topology.second = 0 ; + s_core_capacity = 0 ; + s_hwloc_topology = 0 ; + s_hwloc_location = 0 ; + s_process_binding = 0 ; +} + +Sentinel::Sentinel() +{ +#if defined(__MIC__) + static const bool remove_core_0 = true ; +#else + static const bool remove_core_0 = false ; +#endif + + s_core_topology = std::pair(0,0); + s_core_capacity = 0 ; + s_hwloc_topology = 0 ; + s_hwloc_location = 0 ; + s_process_binding = 0 ; + + for ( unsigned i = 0 ; i < MAX_CORE ; ++i ) s_core[i] = 0 ; + + hwloc_topology_init( & s_hwloc_topology ); + hwloc_topology_load( s_hwloc_topology ); + + s_hwloc_location = hwloc_bitmap_alloc(); + s_process_binding = hwloc_bitmap_alloc(); + + hwloc_get_cpubind( s_hwloc_topology , s_process_binding , HWLOC_CPUBIND_PROCESS ); + + if ( remove_core_0 ) { + + const hwloc_obj_t core = hwloc_get_obj_by_type( s_hwloc_topology , HWLOC_OBJ_CORE , 0 ); + + if ( hwloc_bitmap_intersects( s_process_binding , core->allowed_cpuset ) ) { + + hwloc_bitmap_t s_process_no_core_zero = hwloc_bitmap_alloc(); + + hwloc_bitmap_andnot( s_process_no_core_zero , s_process_binding , core->allowed_cpuset ); + + bool ok = 0 == hwloc_set_cpubind( s_hwloc_topology , + s_process_no_core_zero , + HWLOC_CPUBIND_PROCESS | HWLOC_CPUBIND_STRICT ); + + if ( ok ) { + hwloc_get_cpubind( s_hwloc_topology , s_process_binding , HWLOC_CPUBIND_PROCESS ); + + ok = 0 != hwloc_bitmap_isequal( s_process_binding , s_process_no_core_zero ); + } + + hwloc_bitmap_free( s_process_no_core_zero ); + + if ( ! ok ) { + std::cerr << "WARNING: Kokkos::hwloc attempted and failed to move process off of core #0" << std::endl ; + } + } + } + + // Choose a hwloc object type for the NUMA level, which may not exist. + + hwloc_obj_type_t root_type = HWLOC_OBJ_TYPE_MAX ; + + { + // Object types to search, in order. + static const hwloc_obj_type_t candidate_root_type[] = + { HWLOC_OBJ_NODE /* NUMA region */ + , HWLOC_OBJ_SOCKET /* hardware socket */ + , HWLOC_OBJ_MACHINE /* local machine */ + }; + + enum { CANDIDATE_ROOT_TYPE_COUNT = + sizeof(candidate_root_type) / sizeof(hwloc_obj_type_t) }; + + for ( int k = 0 ; k < CANDIDATE_ROOT_TYPE_COUNT && HWLOC_OBJ_TYPE_MAX == root_type ; ++k ) { + if ( 0 < hwloc_get_nbobjs_by_type( s_hwloc_topology , candidate_root_type[k] ) ) { + root_type = candidate_root_type[k] ; + } + } + } + + // Determine which of these 'root' types are available to this process. + // The process may have been bound (e.g., by MPI) to a subset of these root types. + // Determine current location of the master (calling) process> + + hwloc_bitmap_t proc_cpuset_location = hwloc_bitmap_alloc(); + + hwloc_get_last_cpu_location( s_hwloc_topology , proc_cpuset_location , HWLOC_CPUBIND_THREAD ); + + const unsigned max_root = hwloc_get_nbobjs_by_type( s_hwloc_topology , root_type ); + + unsigned root_base = max_root ; + unsigned root_count = 0 ; + unsigned core_per_root = 0 ; + unsigned pu_per_core = 0 ; + bool symmetric = true ; + + for ( unsigned i = 0 ; i < max_root ; ++i ) { + + const hwloc_obj_t root = hwloc_get_obj_by_type( s_hwloc_topology , root_type , i ); + + if ( hwloc_bitmap_intersects( s_process_binding , root->allowed_cpuset ) ) { + + ++root_count ; + + // Remember which root (NUMA) object the master thread is running on. + // This will be logical NUMA rank #0 for this process. + + if ( hwloc_bitmap_intersects( proc_cpuset_location, root->allowed_cpuset ) ) { + root_base = i ; + } + + // Count available cores: + + const unsigned max_core = + hwloc_get_nbobjs_inside_cpuset_by_type( s_hwloc_topology , + root->allowed_cpuset , + HWLOC_OBJ_CORE ); + + unsigned core_count = 0 ; + + for ( unsigned j = 0 ; j < max_core ; ++j ) { + + const hwloc_obj_t core = + hwloc_get_obj_inside_cpuset_by_type( s_hwloc_topology , + root->allowed_cpuset , + HWLOC_OBJ_CORE , j ); + + // If process' cpuset intersects core's cpuset then process can access this core. + // Must use intersection instead of inclusion because the Intel-Phi + // MPI may bind the process to only one of the core's hyperthreads. + // + // Assumption: if the process can access any hyperthread of the core + // then it has ownership of the entire core. + // This assumes that it would be performance-detrimental + // to spawn more than one MPI process per core and use nested threading. + + if ( hwloc_bitmap_intersects( s_process_binding , core->allowed_cpuset ) ) { + + ++core_count ; + + const unsigned pu_count = + hwloc_get_nbobjs_inside_cpuset_by_type( s_hwloc_topology , + core->allowed_cpuset , + HWLOC_OBJ_PU ); + + if ( pu_per_core == 0 ) pu_per_core = pu_count ; + + // Enforce symmetry by taking the minimum: + + pu_per_core = std::min( pu_per_core , pu_count ); + + if ( pu_count != pu_per_core ) symmetric = false ; + } + } + + if ( 0 == core_per_root ) core_per_root = core_count ; + + // Enforce symmetry by taking the minimum: + + core_per_root = std::min( core_per_root , core_count ); + + if ( core_count != core_per_root ) symmetric = false ; + } + } + + s_core_topology.first = root_count ; + s_core_topology.second = core_per_root ; + s_core_capacity = pu_per_core ; + + // Fill the 's_core' array for fast mapping from a core coordinate to the + // hwloc cpuset object required for thread location querying and binding. + + for ( unsigned i = 0 ; i < max_root ; ++i ) { + + const unsigned root_rank = ( i + root_base ) % max_root ; + + const hwloc_obj_t root = hwloc_get_obj_by_type( s_hwloc_topology , root_type , root_rank ); + + if ( hwloc_bitmap_intersects( s_process_binding , root->allowed_cpuset ) ) { + + const unsigned max_core = + hwloc_get_nbobjs_inside_cpuset_by_type( s_hwloc_topology , + root->allowed_cpuset , + HWLOC_OBJ_CORE ); + + unsigned core_count = 0 ; + + for ( unsigned j = 0 ; j < max_core && core_count < core_per_root ; ++j ) { + + const hwloc_obj_t core = + hwloc_get_obj_inside_cpuset_by_type( s_hwloc_topology , + root->allowed_cpuset , + HWLOC_OBJ_CORE , j ); + + if ( hwloc_bitmap_intersects( s_process_binding , core->allowed_cpuset ) ) { + + s_core[ core_count + core_per_root * i ] = core->allowed_cpuset ; + + ++core_count ; + } + } + } + } + + hwloc_bitmap_free( proc_cpuset_location ); + + if ( ! symmetric ) { + std::cout << "Kokkos::hwloc WARNING: Using a symmetric subset of a non-symmetric core topology." + << std::endl ; + } +} + + +} // namespace + +//---------------------------------------------------------------------------- +//---------------------------------------------------------------------------- + +bool available() +{ return true ; } + +unsigned get_available_numa_count() +{ sentinel(); return s_core_topology.first ; } + +unsigned get_available_cores_per_numa() +{ sentinel(); return s_core_topology.second ; } + +unsigned get_available_threads_per_core() +{ sentinel(); return s_core_capacity ; } + +//---------------------------------------------------------------------------- +//---------------------------------------------------------------------------- + +unsigned bind_this_thread( + const unsigned coordinate_count , + std::pair coordinate[] ) +{ + unsigned i = 0 ; + + try { + const std::pair current = get_this_thread_coordinate(); + + // Match one of the requests: + for ( i = 0 ; i < coordinate_count && current != coordinate[i] ; ++i ); + + if ( coordinate_count == i ) { + // Match the first request (typically NUMA): + for ( i = 0 ; i < coordinate_count && current.first != coordinate[i].first ; ++i ); + } + + if ( coordinate_count == i ) { + // Match any unclaimed request: + for ( i = 0 ; i < coordinate_count && ~0u == coordinate[i].first ; ++i ); + } + + if ( coordinate_count == i || ! bind_this_thread( coordinate[i] ) ) { + // Failed to bind: + i = ~0u ; + } + + if ( i < coordinate_count ) { + +#if DEBUG_PRINT + if ( current != coordinate[i] ) { + std::cout << " bind_this_thread: rebinding from (" + << current.first << "," + << current.second + << ") to (" + << coordinate[i].first << "," + << coordinate[i].second + << ")" << std::endl ; + } +#endif + + coordinate[i].first = ~0u ; + coordinate[i].second = ~0u ; + } + } + catch( ... ) { + i = ~0u ; + } + + return i ; +} + + +bool bind_this_thread( const std::pair coord ) +{ + if ( ! sentinel() ) return false ; + +#if DEBUG_PRINT + + std::cout << "Kokkos::bind_this_thread() at " ; + + hwloc_get_last_cpu_location( s_hwloc_topology , + s_hwloc_location , HWLOC_CPUBIND_THREAD ); + + print_bitmap( std::cout , s_hwloc_location ); + + std::cout << " to " ; + + print_bitmap( std::cout , s_core[ coord.second + coord.first * s_core_topology.second ] ); + + std::cout << std::endl ; + +#endif + + // As safe and fast as possible. + // Fast-lookup by caching the coordinate -> hwloc cpuset mapping in 's_core'. + return coord.first < s_core_topology.first && + coord.second < s_core_topology.second && + 0 == hwloc_set_cpubind( s_hwloc_topology , + s_core[ coord.second + coord.first * s_core_topology.second ] , + HWLOC_CPUBIND_THREAD | HWLOC_CPUBIND_STRICT ); +} + +bool unbind_this_thread() +{ + if ( ! sentinel() ) return false ; + +#define HWLOC_DEBUG_PRINT 0 + +#if HWLOC_DEBUG_PRINT + + std::cout << "Kokkos::unbind_this_thread() from " ; + + hwloc_get_cpubind( s_hwloc_topology , s_hwloc_location , HWLOC_CPUBIND_THREAD ); + + print_bitmap( std::cout , s_hwloc_location ); + +#endif + + const bool result = + s_hwloc_topology && + 0 == hwloc_set_cpubind( s_hwloc_topology , + s_process_binding , + HWLOC_CPUBIND_THREAD | HWLOC_CPUBIND_STRICT ); + +#if HWLOC_DEBUG_PRINT + + std::cout << " to " ; + + hwloc_get_cpubind( s_hwloc_topology , s_hwloc_location , HWLOC_CPUBIND_THREAD ); + + print_bitmap( std::cout , s_hwloc_location ); + + std::cout << std::endl ; + +#endif + + return result ; + +#undef HWLOC_DEBUG_PRINT + +} + +//---------------------------------------------------------------------------- + +std::pair get_this_thread_coordinate() +{ + std::pair coord(0u,0u); + + if ( ! sentinel() ) return coord ; + + const unsigned n = s_core_topology.first * s_core_topology.second ; + + // Using the pre-allocated 's_hwloc_location' to avoid memory + // allocation by this thread. This call is NOT thread-safe. + hwloc_get_last_cpu_location( s_hwloc_topology , + s_hwloc_location , HWLOC_CPUBIND_THREAD ); + + unsigned i = 0 ; + + while ( i < n && ! hwloc_bitmap_intersects( s_hwloc_location , s_core[ i ] ) ) ++i ; + + if ( i < n ) { + coord.first = i / s_core_topology.second ; + coord.second = i % s_core_topology.second ; + } + + return coord ; +} + +//---------------------------------------------------------------------------- + +} /* namespace hwloc */ +} /* namespace Kokkos */ + +//---------------------------------------------------------------------------- +//---------------------------------------------------------------------------- + +#else /* ! defined( KOKKOS_HAVE_HWLOC ) */ + +namespace Kokkos { +namespace hwloc { + +bool available() { return false ; } + +unsigned get_available_numa_count() { return 1 ; } +unsigned get_available_cores_per_numa() { return 1 ; } +unsigned get_available_threads_per_core() { return 1 ; } + +unsigned bind_this_thread( const unsigned , std::pair[] ) +{ return ~0 ; } + +bool bind_this_thread( const std::pair ) +{ return false ; } + +bool unbind_this_thread() +{ return true ; } + +std::pair get_this_thread_coordinate() +{ return std::pair(0,0); } + +} // namespace hwloc +} // namespace Kokkos + +//---------------------------------------------------------------------------- +//---------------------------------------------------------------------------- + +#endif + + diff --git a/lib/kokkos/core/src/impl/Kokkos_spinwait.cpp b/lib/kokkos/core/src/impl/Kokkos_spinwait.cpp new file mode 100755 index 0000000000..1e9ff91c29 --- /dev/null +++ b/lib/kokkos/core/src/impl/Kokkos_spinwait.cpp @@ -0,0 +1,80 @@ +/* +//@HEADER +// ************************************************************************ +// +// Kokkos: Manycore Performance-Portable Multidimensional Arrays +// Copyright (2012) Sandia Corporation +// +// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +// the U.S. Government retains certain rights in this software. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// 1. Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// 3. Neither the name of the Corporation nor the names of the +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Questions? Contact H. Carter Edwards (hcedwar@sandia.gov) +// +// ************************************************************************ +//@HEADER +*/ + +#include +#include + +/*--------------------------------------------------------------------------*/ + +#if ( KOKKOS_ENABLE_ASM ) + #if defined( __arm__ ) + /* No-operation instruction to idle the thread. */ + #define YIELD asm volatile("nop") + #else + /* Pause instruction to prevent excess processor bus usage */ + #define YIELD asm volatile("pause\n":::"memory") + #endif +#elif defined( KOKKOS_HAVE_WINTHREAD ) + #include + #define YIELD Sleep(0) +#else + #include + #define YIELD sched_yield() +#endif + +/*--------------------------------------------------------------------------*/ + +namespace Kokkos { +namespace Impl { +#if defined( KOKKOS_ACTIVE_EXECUTION_MEMORY_SPACE_HOST ) +void spinwait( volatile int & flag , const int value ) +{ + while ( value == flag ) { + YIELD ; + } +} +#endif + +} /* namespace Impl */ +} /* namespace Kokkos */ + diff --git a/lib/kokkos/core/src/impl/Kokkos_spinwait.hpp b/lib/kokkos/core/src/impl/Kokkos_spinwait.hpp new file mode 100755 index 0000000000..966291abd9 --- /dev/null +++ b/lib/kokkos/core/src/impl/Kokkos_spinwait.hpp @@ -0,0 +1,64 @@ +/* +//@HEADER +// ************************************************************************ +// +// Kokkos: Manycore Performance-Portable Multidimensional Arrays +// Copyright (2012) Sandia Corporation +// +// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +// the U.S. Government retains certain rights in this software. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// 1. Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// 3. Neither the name of the Corporation nor the names of the +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Questions? Contact H. Carter Edwards (hcedwar@sandia.gov) +// +// ************************************************************************ +//@HEADER +*/ + + +#ifndef KOKKOS_SPINWAIT_HPP +#define KOKKOS_SPINWAIT_HPP + +#include + +namespace Kokkos { +namespace Impl { + +#if defined( KOKKOS_ACTIVE_EXECUTION_MEMORY_SPACE_HOST ) +void spinwait( volatile int & flag , const int value ); +#else +KOKKOS_INLINE_FUNCTION +void spinwait( volatile int & , const int ) {} +#endif + +} /* namespace Impl */ +} /* namespace Kokkos */ + +#endif /* #ifndef KOKKOS_SPINWAIT_HPP */ +