ICODE-ADC/lammps
4
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Update Kokkos library in LAMMPS to v3.7.0

Browse Source
This commit is contained in:
Stan Gerald Moore
2022-10-04 14:04:40 -06:00
parent dd072f7e08
commit f9f9e44f2d
653 changed files with 41432 additions and 33597 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
lib/kokkos/simd/CMakeLists.txt Normal file
View File

@ -0,0 +1,10 @@
KOKKOS_SUBPACKAGE(Simd)
IF (NOT Kokkos_INSTALL_TESTING)
ADD_SUBDIRECTORY(src)
ENDIF()
KOKKOS_ADD_TEST_DIRECTORIES(unit_tests)
KOKKOS_SUBPACKAGE_POSTPROCESS()

5
lib/kokkos/simd/cmake/Dependencies.cmake Normal file
View File

@ -0,0 +1,5 @@
TRIBITS_PACKAGE_DEFINE_DEPENDENCIES(
LIB_REQUIRED_PACKAGES KokkosCore
LIB_OPTIONAL_TPLS Pthread CUDA HWLOC HPX
TEST_OPTIONAL_TPLS CUSPARSE
)

29
lib/kokkos/simd/src/CMakeLists.txt Normal file
View File

@ -0,0 +1,29 @@
#I have to leave these here for tribits
KOKKOS_INCLUDE_DIRECTORIES(${CMAKE_CURRENT_BINARY_DIR})
KOKKOS_INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR})
#-----------------------------------------------------------------------------
FILE(GLOB SIMD_HEADERS *.hpp)
FILE(GLOB SIMD_SOURCES *.cpp)
INSTALL (
DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/"
DESTINATION ${KOKKOS_HEADER_DIR}
FILES_MATCHING PATTERN "*.hpp"
)
#-----------------------------------------------------------------------------
# We have to pass the sources in here for Tribits
# These will get ignored for standalone CMake and a true interface library made
KOKKOS_ADD_LIBRARY(
kokkossimd
SOURCES ${SIMD_SOURCES}
HEADERS ${SIMD_HEADERS}
)
KOKKOS_LIB_INCLUDE_DIRECTORIES(kokkossimd
${KOKKOS_TOP_BUILD_DIR}
${CMAKE_CURRENT_BINARY_DIR}
${CMAKE_CURRENT_SOURCE_DIR}
)

161
lib/kokkos/simd/src/Kokkos_SIMD.hpp Normal file
View File

@ -0,0 +1,161 @@
/*
//@HEADER
// ************************************************************************
//
// Kokkos v. 3.0
// Copyright (2020) National Technology & Engineering
// Solutions of Sandia, LLC (NTESS).
//
// Under the terms of Contract DE-NA0003525 with NTESS,
// the U.S. Government retains certain rights in this software.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
// 1. Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
//
// 2. Redistributions in binary form must reproduce the above copyright
// notice, this list of conditions and the following disclaimer in the
// documentation and/or other materials provided with the distribution.
//
// 3. Neither the name of the Corporation nor the names of the
// contributors may be used to endorse or promote products derived from
// this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY
// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE
// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
//
// Questions? Contact Christian R. Trott (crtrott@sandia.gov)
//
// ************************************************************************
//@HEADER
*/
#ifndef KOKKOS_SIMD_HPP
#define KOKKOS_SIMD_HPP
#include <Kokkos_SIMD_Common.hpp>
#include <Kokkos_SIMD_Scalar.hpp>
#ifdef KOKKOS_ARCH_AVX512XEON
#include <Kokkos_SIMD_AVX512.hpp>
#endif
namespace Kokkos {
namespace Experimental {
namespace simd_abi {
namespace Impl {
#if defined(KOKKOS_ARCH_AVX512XEON)
using host_native = avx512_fixed_size<8>;
#else
using host_native = scalar;
#endif
template <class T>
struct ForSpace;
#ifdef KOKKOS_ENABLE_SERIAL
template <>
struct ForSpace<Kokkos::Serial> {
using type = host_native;
};
#endif
#ifdef KOKKOS_ENABLE_CUDA
template <>
struct ForSpace<Kokkos::Cuda> {
using type = scalar;
};
#endif
#ifdef KOKKOS_ENABLE_THREADS
template <>
struct ForSpace<Kokkos::Threads> {
using type = host_native;
};
#endif
#ifdef KOKKOS_ENABLE_HPX
template <>
struct ForSpace<Kokkos::Experimental::HPX> {
using type = scalar;
};
#endif
#ifdef KOKKOS_ENABLE_OPENMP
template <>
struct ForSpace<Kokkos::OpenMP> {
using type = host_native;
};
#endif
#ifdef KOKKOS_ENABLE_OPENMPTARGET
template <>
struct ForSpace<Kokkos::Experimental::OpenMPTarget> {
using type = scalar;
};
#endif
#ifdef KOKKOS_ENABLE_HIP
template <>
struct ForSpace<Kokkos::Experimental::HIP> {
using type = scalar;
};
#endif
#ifdef KOKKOS_ENABLE_SYCL
template <>
struct ForSpace<Kokkos::Experimental::SYCL> {
using type = scalar;
};
#endif
} // namespace Impl
template <class Space>
using ForSpace = typename Impl::ForSpace<typename Space::execution_space>::type;
template <class T>
using native = ForSpace<Kokkos::DefaultExecutionSpace>;
} // namespace simd_abi
template <class T>
using native_simd = simd<T, simd_abi::native<T>>;
template <class T>
using native_simd_mask = simd_mask<T, simd_abi::native<T>>;
namespace Impl {
template <class... Abis>
class abi_set {};
#ifdef KOKKOS_ARCH_AVX512XEON
using host_abi_set = abi_set<simd_abi::scalar, simd_abi::avx512_fixed_size<8>>;
#else
using host_abi_set = abi_set<simd_abi::scalar>;
#endif
using device_abi_set = abi_set<simd_abi::scalar>;
} // namespace Impl
} // namespace Experimental
} // namespace Kokkos
#endif

1023
lib/kokkos/simd/src/Kokkos_SIMD_AVX512.hpp Normal file
View File

File diff suppressed because it is too large Load Diff

428
lib/kokkos/simd/src/Kokkos_SIMD_Common.hpp Normal file
View File

@ -0,0 +1,428 @@
/*
//@HEADER
// ************************************************************************
//
// Kokkos v. 3.0
// Copyright (2020) National Technology & Engineering
// Solutions of Sandia, LLC (NTESS).
//
// Under the terms of Contract DE-NA0003525 with NTESS,
// the U.S. Government retains certain rights in this software.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
// 1. Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
//
// 2. Redistributions in binary form must reproduce the above copyright
// notice, this list of conditions and the following disclaimer in the
// documentation and/or other materials provided with the distribution.
//
// 3. Neither the name of the Corporation nor the names of the
// contributors may be used to endorse or promote products derived from
// this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY
// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE
// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
//
// Questions? Contact Christian R. Trott (crtrott@sandia.gov)
//
// ************************************************************************
//@HEADER
*/
#ifndef KOKKOS_SIMD_COMMON_HPP
#define KOKKOS_SIMD_COMMON_HPP
#include <cmath>
#include <cstring>
#include <Kokkos_Core.hpp>
namespace Kokkos {
namespace Experimental {
template <class To, class From>
[[nodiscard]] KOKKOS_FORCEINLINE_FUNCTION constexpr To bit_cast(
From const& src) {
To dst;
std::memcpy(&dst, &src, sizeof(To));
return dst;
}
template <class T, class Abi>
class simd;
template <class T, class Abi>
class simd_mask;
struct element_aligned_tag {};
// class template declarations for const_where_expression and where_expression
template <class M, class T>
class const_where_expression {
protected:
T& m_value;
M const& m_mask;
public:
const_where_expression(M const& mask_arg, T const& value_arg)
: m_value(const_cast<T&>(value_arg)), m_mask(mask_arg) {}
KOKKOS_FORCEINLINE_FUNCTION T const& value() const { return this->m_value; }
};
template <class M, class T>
class where_expression : public const_where_expression<M, T> {
using base_type = const_where_expression<M, T>;
public:
where_expression(M const& mask_arg, T& value_arg)
: base_type(mask_arg, value_arg) {}
KOKKOS_FORCEINLINE_FUNCTION T& value() { return this->m_value; }
};
// specializations of where expression templates for the case when the
// mask type is bool, to allow generic code to use where() on both
// SIMD types and non-SIMD builtin arithmetic types
template <class T>
class const_where_expression<bool, T> {
protected:
T& m_value;
bool m_mask;
public:
KOKKOS_FORCEINLINE_FUNCTION
const_where_expression(bool mask_arg, T const& value_arg)
: m_value(const_cast<T&>(value_arg)), m_mask(mask_arg) {}
KOKKOS_FORCEINLINE_FUNCTION T const& value() const { return this->m_value; }
};
template <class T>
class where_expression<bool, T> : public const_where_expression<bool, T> {
using base_type = const_where_expression<bool, T>;
public:
KOKKOS_FORCEINLINE_FUNCTION
where_expression(bool mask_arg, T& value_arg)
: base_type(mask_arg, value_arg) {}
KOKKOS_FORCEINLINE_FUNCTION T& value() { return this->m_value; }
template <class U,
std::enable_if_t<std::is_convertible_v<U, T>, bool> = false>
KOKKOS_FORCEINLINE_FUNCTION void operator=(U const& x) {
if (this->m_mask) this->m_value = x;
}
};
template <class T, class Abi>
[[nodiscard]] KOKKOS_FORCEINLINE_FUNCTION
where_expression<simd_mask<T, Abi>, simd<T, Abi>>
where(typename simd<T, Abi>::mask_type const& mask, simd<T, Abi>& value) {
return where_expression(mask, value);
}
template <class T, class Abi>
[[nodiscard]] KOKKOS_FORCEINLINE_FUNCTION
const_where_expression<simd_mask<T, Abi>, simd<T, Abi>>
where(typename simd<T, Abi>::mask_type const& mask,
simd<T, Abi> const& value) {
return const_where_expression(mask, value);
}
template <class T>
[[nodiscard]] KOKKOS_FORCEINLINE_FUNCTION where_expression<bool, T> where(
bool mask, T& value) {
return where_expression(mask, value);
}
template <class T>
[[nodiscard]] KOKKOS_FORCEINLINE_FUNCTION const_where_expression<bool, T> where(
bool mask, T const& value) {
return const_where_expression(mask, value);
}
// The code below provides:
// operator@(simd<T, Abi>, Arithmetic)
// operator@(Arithmetic, simd<T, Abi>)
// operator@=(simd<T, Abi>&, U&&)
// operator@=(where_expression<M, T>&, U&&)
template <class T, class U, class Abi,
std::enable_if_t<std::is_arithmetic_v<U>, bool> = false>
[[nodiscard]] KOKKOS_FORCEINLINE_FUNCTION auto operator+(
Experimental::simd<T, Abi> const& lhs, U rhs) {
using result_member = decltype(lhs[0] + rhs);
return Experimental::simd<result_member, Abi>(lhs) +
Experimental::simd<result_member, Abi>(rhs);
}
template <class T, class U, class Abi,
std::enable_if_t<std::is_arithmetic_v<U>, bool> = false>
[[nodiscard]] KOKKOS_FORCEINLINE_FUNCTION auto operator+(
U lhs, Experimental::simd<T, Abi> const& rhs) {
using result_member = decltype(lhs + rhs[0]);
return Experimental::simd<result_member, Abi>(lhs) +
Experimental::simd<result_member, Abi>(rhs);
}
template <class T, class U, class Abi>
KOKKOS_FORCEINLINE_FUNCTION simd<T, Abi>& operator+=(simd<T, Abi>& lhs,
U&& rhs) {
lhs = lhs + std::forward<U>(rhs);
return lhs;
}
template <class M, class T, class U>
KOKKOS_FORCEINLINE_FUNCTION where_expression<M, T>& operator+=(
where_expression<M, T>& lhs, U&& rhs) {
lhs = lhs.value() + std::forward<U>(rhs);
return lhs;
}
template <class T, class U, class Abi,
std::enable_if_t<std::is_arithmetic_v<U>, bool> = false>
[[nodiscard]] KOKKOS_FORCEINLINE_FUNCTION auto operator-(
Experimental::simd<T, Abi> const& lhs, U rhs) {
using result_member = decltype(lhs[0] - rhs);
return Experimental::simd<result_member, Abi>(lhs) -
Experimental::simd<result_member, Abi>(rhs);
}
template <class T, class U, class Abi,
std::enable_if_t<std::is_arithmetic_v<U>, bool> = false>
[[nodiscard]] KOKKOS_FORCEINLINE_FUNCTION auto operator-(
U lhs, Experimental::simd<T, Abi> const& rhs) {
using result_member = decltype(lhs - rhs[0]);
return Experimental::simd<result_member, Abi>(lhs) -
Experimental::simd<result_member, Abi>(rhs);
}
template <class T, class U, class Abi>
KOKKOS_FORCEINLINE_FUNCTION simd<T, Abi>& operator-=(simd<T, Abi>& lhs,
U&& rhs) {
lhs = lhs - std::forward<U>(rhs);
return lhs;
}
template <class M, class T, class U>
KOKKOS_FORCEINLINE_FUNCTION where_expression<M, T>& operator-=(
where_expression<M, T>& lhs, U&& rhs) {
lhs = lhs.value() - std::forward<U>(rhs);
return lhs;
}
template <class T, class U, class Abi,
std::enable_if_t<std::is_arithmetic_v<U>, bool> = false>
[[nodiscard]] KOKKOS_FORCEINLINE_FUNCTION auto operator*(
Experimental::simd<T, Abi> const& lhs, U rhs) {
using result_member = decltype(lhs[0] * rhs);
return Experimental::simd<result_member, Abi>(lhs) *
Experimental::simd<result_member, Abi>(rhs);
}
template <class T, class U, class Abi,
std::enable_if_t<std::is_arithmetic_v<U>, bool> = false>
[[nodiscard]] KOKKOS_FORCEINLINE_FUNCTION auto operator*(
U lhs, Experimental::simd<T, Abi> const& rhs) {
using result_member = decltype(lhs * rhs[0]);
return Experimental::simd<result_member, Abi>(lhs) *
Experimental::simd<result_member, Abi>(rhs);
}
template <class T, class U, class Abi>
KOKKOS_FORCEINLINE_FUNCTION simd<T, Abi>& operator*=(simd<T, Abi>& lhs,
U&& rhs) {
lhs = lhs * std::forward<U>(rhs);
return lhs;
}
template <class M, class T, class U>
KOKKOS_FORCEINLINE_FUNCTION where_expression<M, T>& operator*=(
where_expression<M, T>& lhs, U&& rhs) {
lhs = lhs.value() * std::forward<U>(rhs);
return lhs;
}
template <class T, class U, class Abi,
std::enable_if_t<std::is_arithmetic_v<U>, bool> = false>
[[nodiscard]] KOKKOS_FORCEINLINE_FUNCTION auto operator/(
Experimental::simd<T, Abi> const& lhs, U rhs) {
using result_member = decltype(lhs[0] / rhs);
return Experimental::simd<result_member, Abi>(lhs) /
Experimental::simd<result_member, Abi>(rhs);
}
template <class T, class U, class Abi,
std::enable_if_t<std::is_arithmetic_v<U>, bool> = false>
[[nodiscard]] KOKKOS_FORCEINLINE_FUNCTION auto operator/(
U lhs, Experimental::simd<T, Abi> const& rhs) {
using result_member = decltype(lhs / rhs[0]);
return Experimental::simd<result_member, Abi>(lhs) /
Experimental::simd<result_member, Abi>(rhs);
}
template <class T, class U, class Abi>
KOKKOS_FORCEINLINE_FUNCTION simd<T, Abi>& operator/=(simd<T, Abi>& lhs,
U&& rhs) {
lhs = lhs / std::forward<U>(rhs);
return lhs;
}
template <class M, class T, class U>
KOKKOS_FORCEINLINE_FUNCTION where_expression<M, T>& operator/=(
where_expression<M, T>& lhs, U&& rhs) {
lhs = lhs.value() / std::forward<U>(rhs);
return lhs;
}
// implement mask reductions for type bool to allow generic code to accept
// both simd<double, Abi> and just double
[[nodiscard]] KOKKOS_FORCEINLINE_FUNCTION constexpr bool all_of(bool a) {
return a;
}
[[nodiscard]] KOKKOS_FORCEINLINE_FUNCTION constexpr bool any_of(bool a) {
return a;
}
[[nodiscard]] KOKKOS_FORCEINLINE_FUNCTION constexpr bool none_of(bool a) {
return !a;
}
// fallback implementations of reductions across simd_mask:
template <class T, class Abi>
[[nodiscard]] KOKKOS_FORCEINLINE_FUNCTION bool all_of(
simd_mask<T, Abi> const& a) {
return a == simd_mask<T, Abi>(true);
}
template <class T, class Abi>
[[nodiscard]] KOKKOS_FORCEINLINE_FUNCTION bool any_of(
simd_mask<T, Abi> const& a) {
return a != simd_mask<T, Abi>(false);
}
template <class T, class Abi>
[[nodiscard]] KOKKOS_FORCEINLINE_FUNCTION bool none_of(
simd_mask<T, Abi> const& a) {
return a == simd_mask<T, Abi>(false);
}
} // namespace Experimental
template <class T, class Abi>
[[nodiscard]] KOKKOS_FORCEINLINE_FUNCTION Experimental::simd<T, Abi> min(
Experimental::simd<T, Abi> const& a, Experimental::simd<T, Abi> const& b) {
Experimental::simd<T, Abi> result;
for (std::size_t i = 0; i < Experimental::simd<T, Abi>::size(); ++i) {
result[i] = Kokkos::min(a[i], b[i]);
}
return result;
}
template <class T, class Abi>
[[nodiscard]] KOKKOS_FORCEINLINE_FUNCTION Experimental::simd<T, Abi> max(
Experimental::simd<T, Abi> const& a, Experimental::simd<T, Abi> const& b) {
Experimental::simd<T, Abi> result;
for (std::size_t i = 0; i < Experimental::simd<T, Abi>::size(); ++i) {
result[i] = Kokkos::max(a[i], b[i]);
}
return result;
}
// fallback implementations of <cmath> functions.
// individual Abi types may provide overloads with more efficient
// implementations.
// These are not in the Experimental namespace because their double
// overloads are not either
#define KOKKOS_IMPL_SIMD_UNARY_FUNCTION(FUNC) \
template <class Abi> \
[[nodiscard]] KOKKOS_FORCEINLINE_FUNCTION Experimental::simd<double, Abi> \
FUNC(Experimental::simd<double, Abi> const& a) { \
Experimental::simd<double, Abi> result; \
for (std::size_t i = 0; i < Experimental::simd<double, Abi>::size(); \
++i) { \
result[i] = Kokkos::FUNC(a[i]); \
} \
return result; \
}
KOKKOS_IMPL_SIMD_UNARY_FUNCTION(abs)
KOKKOS_IMPL_SIMD_UNARY_FUNCTION(exp)
KOKKOS_IMPL_SIMD_UNARY_FUNCTION(exp2)
KOKKOS_IMPL_SIMD_UNARY_FUNCTION(log)
KOKKOS_IMPL_SIMD_UNARY_FUNCTION(log10)
KOKKOS_IMPL_SIMD_UNARY_FUNCTION(log2)
KOKKOS_IMPL_SIMD_UNARY_FUNCTION(sqrt)
KOKKOS_IMPL_SIMD_UNARY_FUNCTION(cbrt)
KOKKOS_IMPL_SIMD_UNARY_FUNCTION(sin)
KOKKOS_IMPL_SIMD_UNARY_FUNCTION(cos)
KOKKOS_IMPL_SIMD_UNARY_FUNCTION(tan)
KOKKOS_IMPL_SIMD_UNARY_FUNCTION(asin)
KOKKOS_IMPL_SIMD_UNARY_FUNCTION(acos)
KOKKOS_IMPL_SIMD_UNARY_FUNCTION(atan)
KOKKOS_IMPL_SIMD_UNARY_FUNCTION(sinh)
KOKKOS_IMPL_SIMD_UNARY_FUNCTION(cosh)
KOKKOS_IMPL_SIMD_UNARY_FUNCTION(tanh)
KOKKOS_IMPL_SIMD_UNARY_FUNCTION(asinh)
KOKKOS_IMPL_SIMD_UNARY_FUNCTION(acosh)
KOKKOS_IMPL_SIMD_UNARY_FUNCTION(atanh)
KOKKOS_IMPL_SIMD_UNARY_FUNCTION(erf)
KOKKOS_IMPL_SIMD_UNARY_FUNCTION(erfc)
KOKKOS_IMPL_SIMD_UNARY_FUNCTION(tgamma)
KOKKOS_IMPL_SIMD_UNARY_FUNCTION(lgamma)
#define KOKKOS_IMPL_SIMD_BINARY_FUNCTION(FUNC) \
template <class Abi> \
[[nodiscard]] KOKKOS_FORCEINLINE_FUNCTION Experimental::simd<double, Abi> \
FUNC(Experimental::simd<double, Abi> const& a, \
Experimental::simd<double, Abi> const& b) { \
Experimental::simd<double, Abi> result; \
for (std::size_t i = 0; i < Experimental::simd<double, Abi>::size(); \
++i) { \
result[i] = Kokkos::FUNC(a[i], b[i]); \
} \
return result; \
}
KOKKOS_IMPL_SIMD_BINARY_FUNCTION(pow)
KOKKOS_IMPL_SIMD_BINARY_FUNCTION(hypot)
KOKKOS_IMPL_SIMD_BINARY_FUNCTION(atan2)
KOKKOS_IMPL_SIMD_BINARY_FUNCTION(copysign)
#define KOKKOS_IMPL_SIMD_TERNARY_FUNCTION(FUNC) \
template <class Abi> \
[[nodiscard]] KOKKOS_FORCEINLINE_FUNCTION Experimental::simd<double, Abi> \
FUNC(Experimental::simd<double, Abi> const& a, \
Experimental::simd<double, Abi> const& b, \
Experimental::simd<double, Abi> const& c) { \
Experimental::simd<double, Abi> result; \
for (std::size_t i = 0; i < Experimental::simd<double, Abi>::size(); \
++i) { \
result[i] = Kokkos::FUNC(a[i], b[i], c[i]); \
} \
return result; \
}
KOKKOS_IMPL_SIMD_TERNARY_FUNCTION(fma)
KOKKOS_IMPL_SIMD_TERNARY_FUNCTION(hypot)
} // namespace Kokkos
#endif

353
lib/kokkos/simd/src/Kokkos_SIMD_Scalar.hpp Normal file
View File

@ -0,0 +1,353 @@
/*
//@HEADER
// ************************************************************************
//
// Kokkos v. 3.0
// Copyright (2020) National Technology & Engineering
// Solutions of Sandia, LLC (NTESS).
//
// Under the terms of Contract DE-NA0003525 with NTESS,
// the U.S. Government retains certain rights in this software.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
// 1. Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
//
// 2. Redistributions in binary form must reproduce the above copyright
// notice, this list of conditions and the following disclaimer in the
// documentation and/or other materials provided with the distribution.
//
// 3. Neither the name of the Corporation nor the names of the
// contributors may be used to endorse or promote products derived from
// this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY
// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE
// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
//
// Questions? Contact Christian R. Trott (crtrott@sandia.gov)
//
// ************************************************************************
//@HEADER
*/
#ifndef KOKKOS_SIMD_SCALAR_HPP
#define KOKKOS_SIMD_SCALAR_HPP
#include <type_traits>
#include <climits>
#include <cfloat>
#include <Kokkos_SIMD_Common.hpp>
namespace Kokkos {
namespace Experimental {
namespace simd_abi {
class scalar {};
} // namespace simd_abi
template <class T>
class simd_mask<T, simd_abi::scalar> {
bool m_value;
public:
using value_type = bool;
using simd_type = simd<T, simd_abi::scalar>;
using abi_type = simd_abi::scalar;
using reference = value_type&;
KOKKOS_DEFAULTED_FUNCTION simd_mask() = default;
KOKKOS_FORCEINLINE_FUNCTION static constexpr std::size_t size() { return 1; }
KOKKOS_FORCEINLINE_FUNCTION explicit simd_mask(value_type value)
: m_value(value) {}
template <class U>
KOKKOS_FORCEINLINE_FUNCTION simd_mask(
simd_mask<U, simd_abi::scalar> const& other)
: m_value(static_cast<bool>(other)) {}
KOKKOS_FORCEINLINE_FUNCTION constexpr explicit operator bool() const {
return m_value;
}
KOKKOS_FORCEINLINE_FUNCTION reference operator[](std::size_t) {
return m_value;
}
KOKKOS_FORCEINLINE_FUNCTION value_type operator[](std::size_t) const {
return m_value;
}
KOKKOS_FORCEINLINE_FUNCTION simd_mask
operator||(simd_mask const& other) const {
return simd_mask(m_value || other.m_value);
}
KOKKOS_FORCEINLINE_FUNCTION simd_mask
operator&&(simd_mask const& other) const {
return simd_mask(m_value && other.m_value);
}
KOKKOS_FORCEINLINE_FUNCTION simd_mask operator!() const {
return simd_mask(!m_value);
}
KOKKOS_FORCEINLINE_FUNCTION bool operator==(simd_mask const& other) const {
return m_value == other.m_value;
}
KOKKOS_FORCEINLINE_FUNCTION bool operator!=(simd_mask const& other) const {
return m_value != other.m_value;
}
};
template <class T>
class simd<T, simd_abi::scalar> {
T m_value;
public:
using value_type = T;
using abi_type = simd_abi::scalar;
using mask_type = simd_mask<T, abi_type>;
using reference = value_type&;
KOKKOS_DEFAULTED_FUNCTION simd() = default;
KOKKOS_DEFAULTED_FUNCTION simd(simd const&) = default;
KOKKOS_DEFAULTED_FUNCTION simd(simd&&) = default;
KOKKOS_DEFAULTED_FUNCTION simd& operator=(simd const&) = default;
KOKKOS_DEFAULTED_FUNCTION simd& operator=(simd&&) = default;
KOKKOS_FORCEINLINE_FUNCTION static constexpr std::size_t size() { return 1; }
template <class U, std::enable_if_t<std::is_convertible_v<U, value_type>,
bool> = false>
KOKKOS_FORCEINLINE_FUNCTION simd(U&& value) : m_value(value) {}
template <class U, std::enable_if_t<std::is_convertible_v<U, value_type>,
bool> = false>
KOKKOS_FORCEINLINE_FUNCTION explicit simd(simd<U, abi_type> const& other)
: m_value(static_cast<U>(other)) {}
template <class G,
std::enable_if_t<
// basically, can you do { value_type r =
// gen(std::integral_constant<std::size_t, i>()); }
std::is_invocable_r_v<value_type, G,
std::integral_constant<std::size_t, 0>>,
bool> = false>
KOKKOS_FORCEINLINE_FUNCTION simd(G&& gen)
: m_value(gen(std::integral_constant<std::size_t, 0>())) {}
KOKKOS_FORCEINLINE_FUNCTION simd operator-() const { return simd(-m_value); }
KOKKOS_FORCEINLINE_FUNCTION simd operator>>(int rhs) const {
return simd(m_value >> rhs);
}
KOKKOS_FORCEINLINE_FUNCTION simd
operator>>(simd<int, abi_type> const& rhs) const {
return simd(m_value >> static_cast<int>(rhs));
}
KOKKOS_FORCEINLINE_FUNCTION simd operator<<(int rhs) const {
return simd(m_value << rhs);
}
KOKKOS_FORCEINLINE_FUNCTION simd
operator<<(simd<int, abi_type> const& rhs) const {
return simd(m_value << static_cast<int>(rhs));
}
KOKKOS_FORCEINLINE_FUNCTION simd operator&(simd const& other) const {
return m_value & other.m_value;
}
KOKKOS_FORCEINLINE_FUNCTION simd operator|(simd const& other) const {
return m_value | other.m_value;
}
KOKKOS_FORCEINLINE_FUNCTION constexpr explicit operator T() const {
return m_value;
}
KOKKOS_FORCEINLINE_FUNCTION mask_type operator<(simd const& other) const {
return mask_type(m_value < other.m_value);
}
KOKKOS_FORCEINLINE_FUNCTION mask_type operator>(simd const& other) const {
return mask_type(m_value > other.m_value);
}
KOKKOS_FORCEINLINE_FUNCTION mask_type operator<=(simd const& other) const {
return mask_type(m_value <= other.m_value);
}
KOKKOS_FORCEINLINE_FUNCTION mask_type operator>=(simd const& other) const {
return mask_type(m_value >= other.m_value);
}
KOKKOS_FORCEINLINE_FUNCTION mask_type operator==(simd const& other) const {
return mask_type(m_value == other.m_value);
}
KOKKOS_FORCEINLINE_FUNCTION mask_type operator!=(simd const& other) const {
return mask_type(m_value != other.m_value);
}
KOKKOS_FORCEINLINE_FUNCTION void copy_from(T const* ptr,
element_aligned_tag) {
m_value = *ptr;
}
KOKKOS_FORCEINLINE_FUNCTION void copy_to(T* ptr, element_aligned_tag) const {
*ptr = m_value;
}
KOKKOS_FORCEINLINE_FUNCTION reference operator[](std::size_t) {
return m_value;
}
KOKKOS_FORCEINLINE_FUNCTION value_type operator[](std::size_t) const {
return m_value;
}
};
template <class T>
[[nodiscard]] KOKKOS_FORCEINLINE_FUNCTION simd<T, simd_abi::scalar> operator*(
simd<T, simd_abi::scalar> const& lhs,
simd<T, simd_abi::scalar> const& rhs) {
return simd<T, simd_abi::scalar>(static_cast<T>(lhs) * static_cast<T>(rhs));
}
template <class T>
[[nodiscard]] KOKKOS_FORCEINLINE_FUNCTION simd<T, simd_abi::scalar> operator/(
simd<T, simd_abi::scalar> const& lhs,
simd<T, simd_abi::scalar> const& rhs) {
return simd<T, simd_abi::scalar>(static_cast<T>(lhs) / static_cast<T>(rhs));
}
template <class T>
[[nodiscard]] KOKKOS_FORCEINLINE_FUNCTION simd<T, simd_abi::scalar> operator+(
simd<T, simd_abi::scalar> const& lhs,
simd<T, simd_abi::scalar> const& rhs) {
return simd<T, simd_abi::scalar>(static_cast<T>(lhs) + static_cast<T>(rhs));
}
template <class T>
[[nodiscard]] KOKKOS_FORCEINLINE_FUNCTION simd<T, simd_abi::scalar> operator-(
simd<T, simd_abi::scalar> const& lhs,
simd<T, simd_abi::scalar> const& rhs) {
return simd<T, simd_abi::scalar>(static_cast<T>(lhs) - static_cast<T>(rhs));
}
template <class T>
[[nodiscard]] KOKKOS_FORCEINLINE_FUNCTION simd<T, simd_abi::scalar> abs(
simd<T, simd_abi::scalar> const& a) {
return simd<T, simd_abi::scalar>(std::abs(static_cast<T>(a)));
}
template <class T>
[[nodiscard]] KOKKOS_FORCEINLINE_FUNCTION simd<T, simd_abi::scalar> sqrt(
simd<T, simd_abi::scalar> const& a) {
return simd<T, simd_abi::scalar>(std::sqrt(static_cast<T>(a)));
}
template <class T>
KOKKOS_FORCEINLINE_FUNCTION simd<T, simd_abi::scalar> fma(
simd<T, simd_abi::scalar> const& x, simd<T, simd_abi::scalar> const& y,
simd<T, simd_abi::scalar> const& z) {
return simd<T, simd_abi::scalar>((static_cast<T>(x) * static_cast<T>(y)) +
static_cast<T>(z));
}
template <class T>
KOKKOS_FORCEINLINE_FUNCTION simd<T, simd_abi::scalar> condition(
desul::Impl::dont_deduce_this_parameter_t<
simd_mask<T, simd_abi::scalar>> const& a,
simd<T, simd_abi::scalar> const& b, simd<T, simd_abi::scalar> const& c) {
return simd<T, simd_abi::scalar>(static_cast<bool>(a) ? static_cast<T>(b)
: static_cast<T>(c));
}
template <class T, class Abi>
[[nodiscard]] KOKKOS_FORCEINLINE_FUNCTION simd<T, Abi> copysign(
simd<T, Abi> const& a, simd<T, Abi> const& b) {
return std::copysign(static_cast<T>(a), static_cast<T>(b));
}
template <class T>
class const_where_expression<simd_mask<T, simd_abi::scalar>,
simd<T, simd_abi::scalar>> {
public:
using abi_type = simd_abi::scalar;
using value_type = simd<T, abi_type>;
using mask_type = simd_mask<T, abi_type>;
protected:
value_type& m_value;
mask_type const& m_mask;
public:
KOKKOS_FORCEINLINE_FUNCTION
const_where_expression(mask_type const& mask_arg, value_type const& value_arg)
: m_value(const_cast<value_type&>(value_arg)), m_mask(mask_arg) {}
KOKKOS_FORCEINLINE_FUNCTION
mask_type const& mask() const { return m_mask; }
KOKKOS_FORCEINLINE_FUNCTION
value_type const& value() const { return m_value; }
KOKKOS_FORCEINLINE_FUNCTION
void copy_to(T* mem, element_aligned_tag) const {
if (static_cast<bool>(m_mask)) *mem = static_cast<T>(m_value);
}
template <class Integral>
KOKKOS_FORCEINLINE_FUNCTION std::enable_if_t<std::is_integral_v<Integral>>
scatter_to(T* mem, simd<Integral, simd_abi::scalar> const& index) const {
if (static_cast<bool>(m_mask))
mem[static_cast<Integral>(index)] = static_cast<T>(m_value);
}
};
template <class T>
class where_expression<simd_mask<T, simd_abi::scalar>,
simd<T, simd_abi::scalar>>
: public const_where_expression<simd_mask<T, simd_abi::scalar>,
simd<T, simd_abi::scalar>> {
using base_type = const_where_expression<simd_mask<T, simd_abi::scalar>,
simd<T, simd_abi::scalar>>;
public:
using typename base_type::value_type;
KOKKOS_FORCEINLINE_FUNCTION
where_expression(simd_mask<T, simd_abi::scalar> const& mask_arg,
simd<T, simd_abi::scalar>& value_arg)
: base_type(mask_arg, value_arg) {}
KOKKOS_FORCEINLINE_FUNCTION
void copy_from(T const* mem, element_aligned_tag) {
if (static_cast<bool>(this->m_mask)) this->m_value = *mem;
}
template <class Integral>
KOKKOS_FORCEINLINE_FUNCTION std::enable_if_t<std::is_integral_v<Integral>>
gather_from(T const* mem, simd<Integral, simd_abi::scalar> const& index) {
if (static_cast<bool>(this->m_mask))
this->m_value = mem[static_cast<Integral>(index)];
}
template <class U, std::enable_if_t<
std::is_convertible_v<U, simd<T, simd_abi::scalar>>,
bool> = false>
KOKKOS_FORCEINLINE_FUNCTION void operator=(U&& x) {
if (static_cast<bool>(this->m_mask))
this->m_value =
static_cast<simd<T, simd_abi::scalar>>(std::forward<U>(x));
}
};
template <class T, class BinaryOp>
[[nodiscard]] KOKKOS_FORCEINLINE_FUNCTION T
reduce(const_where_expression<simd_mask<T, simd_abi::scalar>,
simd<T, simd_abi::scalar>> const& x,
T identity_element, BinaryOp) {
return static_cast<bool>(x.mask()) ? static_cast<T>(x.value())
: identity_element;
}
template <class T>
[[nodiscard]] KOKKOS_FORCEINLINE_FUNCTION T
hmax(const_where_expression<simd_mask<T, simd_abi::scalar>,
simd<T, simd_abi::scalar>> const& x) {
return static_cast<bool>(x.mask()) ? static_cast<T>(x.value())
: Kokkos::reduction_identity<T>::max();
}
template <class T>
[[nodiscard]] KOKKOS_FORCEINLINE_FUNCTION T
hmin(const_where_expression<simd_mask<T, simd_abi::scalar>,
simd<T, simd_abi::scalar>> const& x) {
return static_cast<bool>(x.mask()) ? static_cast<T>(x.value())
: Kokkos::reduction_identity<T>::min();
}
} // namespace Experimental
} // namespace Kokkos
#endif

7
lib/kokkos/simd/src/Kokkos_SIMD_dummy.cpp Normal file
View File

@ -0,0 +1,7 @@
// This file is needed in order to get the linker language
// for the header only submodule.
// While we set the language properties in our normal cmake
// path it does not get set in the Trilinos environment.
// Furthermore, setting LINKER_LANGUAGE is only supported
// in CMAKE 3.19 and up.
void KOKKOS_SIMD_SRC_DUMMY_PREVENT_LINK_ERROR() {}

5
lib/kokkos/simd/unit_tests/CMakeLists.txt Normal file
View File

@ -0,0 +1,5 @@
KOKKOS_ADD_EXECUTABLE_AND_TEST(
UnitTest_SIMD
SOURCES
UnitTestMain.cpp
TestSIMD.cpp)

376
lib/kokkos/simd/unit_tests/TestSIMD.cpp Normal file
View File

@ -0,0 +1,376 @@
/*
//@HEADER
// ************************************************************************
//
// Kokkos v. 3.0
// Copyright (2020) National Technology & Engineering
// Solutions of Sandia, LLC (NTESS).
//
// Under the terms of Contract DE-NA0003525 with NTESS,
// the U.S. Government retains certain rights in this software.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
// 1. Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
//
// 2. Redistributions in binary form must reproduce the above copyright
// notice, this list of conditions and the following disclaimer in the
// documentation and/or other materials provided with the distribution.
//
// 3. Neither the name of the Corporation nor the names of the
// contributors may be used to endorse or promote products derived from
// this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY
// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE
// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
//
// Questions? Contact Christian R. Trott (crtrott@sandia.gov)
//
// ************************************************************************
//@HEADER
*/
#include <gtest/gtest.h>
#include <Kokkos_SIMD.hpp>
class gtest_checker {
public:
void truth(bool x) const { EXPECT_TRUE(x); }
template <class T>
void equality(T const& a, T const& b) const {
EXPECT_EQ(a, b);
}
};
class kokkos_checker {
public:
KOKKOS_INLINE_FUNCTION void truth(bool x) const {
if (!x) Kokkos::abort("SIMD unit test truth condition failed on device");
}
template <class T>
KOKKOS_INLINE_FUNCTION void equality(T const& a, T const& b) const {
if (a != b)
Kokkos::abort("SIMD unit test equality condition failed on device");
}
};
template <class T, class Abi>
inline void host_check_equality(
Kokkos::Experimental::simd<T, Abi> const& expected_result,
Kokkos::Experimental::simd<T, Abi> const& computed_result,
std::size_t nlanes) {
gtest_checker checker;
for (std::size_t i = 0; i < nlanes; ++i) {
checker.equality(expected_result[i], computed_result[i]);
}
using mask_type = typename Kokkos::Experimental::simd<T, Abi>::mask_type;
mask_type mask(false);
for (std::size_t i = 0; i < nlanes; ++i) {
mask[i] = true;
}
checker.equality((expected_result == computed_result) && mask, mask);
}
template <class T, class Abi>
KOKKOS_INLINE_FUNCTION void device_check_equality(
Kokkos::Experimental::simd<T, Abi> const& expected_result,
Kokkos::Experimental::simd<T, Abi> const& computed_result,
std::size_t nlanes) {
kokkos_checker checker;
for (std::size_t i = 0; i < nlanes; ++i) {
checker.equality(expected_result[i], computed_result[i]);
}
using mask_type = typename Kokkos::Experimental::simd<T, Abi>::mask_type;
mask_type mask(false);
for (std::size_t i = 0; i < nlanes; ++i) {
mask[i] = true;
}
checker.equality((expected_result == computed_result) && mask, mask);
}
class load_element_aligned {
public:
template <class T, class Abi>
bool host_load(T const* mem, std::size_t n,
Kokkos::Experimental::simd<T, Abi>& result) const {
if (n < result.size()) return false;
result.copy_from(mem, Kokkos::Experimental::element_aligned_tag());
return true;
}
template <class T, class Abi>
KOKKOS_INLINE_FUNCTION bool device_load(
T const* mem, std::size_t n,
Kokkos::Experimental::simd<T, Abi>& result) const {
if (n < result.size()) return false;
result.copy_from(mem, Kokkos::Experimental::element_aligned_tag());
return true;
}
};
class load_masked {
public:
template <class T, class Abi>
bool host_load(T const* mem, std::size_t n,
Kokkos::Experimental::simd<T, Abi>& result) const {
using mask_type = typename Kokkos::Experimental::simd<T, Abi>::mask_type;
mask_type mask(false);
for (std::size_t i = 0; i < n; ++i) {
mask[i] = true;
}
where(mask, result)
.copy_from(mem, Kokkos::Experimental::element_aligned_tag());
where(!mask, result) = 0;
return true;
}
template <class T, class Abi>
KOKKOS_INLINE_FUNCTION bool device_load(
T const* mem, std::size_t n,
Kokkos::Experimental::simd<T, Abi>& result) const {
using mask_type = typename Kokkos::Experimental::simd<T, Abi>::mask_type;
mask_type mask(false);
for (std::size_t i = 0; i < n; ++i) {
mask[i] = true;
}
where(mask, result)
.copy_from(mem, Kokkos::Experimental::element_aligned_tag());
where(!mask, result) = T(0);
return true;
}
};
class load_as_scalars {
public:
template <class T, class Abi>
bool host_load(T const* mem, std::size_t n,
Kokkos::Experimental::simd<T, Abi>& result) const {
for (std::size_t i = 0; i < n; ++i) {
result[i] = mem[i];
}
for (std::size_t i = n; i < result.size(); ++i) {
result[i] = T(0);
}
return true;
}
template <class T, class Abi>
KOKKOS_INLINE_FUNCTION bool device_load(
T const* mem, std::size_t n,
Kokkos::Experimental::simd<T, Abi>& result) const {
for (std::size_t i = 0; i < n; ++i) {
result[i] = mem[i];
}
for (std::size_t i = n; i < result.size(); ++i) {
result[i] = T(0);
}
return true;
}
};
template <class Abi, class Loader, class BinaryOp, class T>
void host_check_binary_op_one_loader(BinaryOp binary_op, std::size_t n,
T const* first_args,
T const* second_args) {
Loader loader;
using simd_type = Kokkos::Experimental::simd<T, Abi>;
std::size_t constexpr width = simd_type::size();
for (std::size_t i = 0; i < n; i += width) {
std::size_t const nremaining = n - i;
std::size_t const nlanes = Kokkos::min(nremaining, width);
simd_type first_arg;
bool const loaded_first_arg =
loader.host_load(first_args + i, nlanes, first_arg);
simd_type second_arg;
bool const loaded_second_arg =
loader.host_load(second_args + i, nlanes, second_arg);
if (!(loaded_first_arg && loaded_second_arg)) continue;
simd_type expected_result;
for (std::size_t lane = 0; lane < nlanes; ++lane) {
expected_result[lane] =
binary_op.on_host(first_arg[lane], second_arg[lane]);
}
simd_type const computed_result = binary_op.on_host(first_arg, second_arg);
host_check_equality(expected_result, computed_result, nlanes);
}
}
template <class Abi, class Loader, class BinaryOp, class T>
KOKKOS_INLINE_FUNCTION void device_check_binary_op_one_loader(
BinaryOp binary_op, std::size_t n, T const* first_args,
T const* second_args) {
Loader loader;
using simd_type = Kokkos::Experimental::simd<T, Abi>;
std::size_t constexpr width = simd_type::size();
for (std::size_t i = 0; i < n; i += width) {
std::size_t const nremaining = n - i;
std::size_t const nlanes = Kokkos::min(nremaining, width);
simd_type first_arg;
bool const loaded_first_arg =
loader.device_load(first_args + i, nlanes, first_arg);
simd_type second_arg;
bool const loaded_second_arg =
loader.device_load(second_args + i, nlanes, second_arg);
if (!(loaded_first_arg && loaded_second_arg)) continue;
simd_type expected_result;
for (std::size_t lane = 0; lane < nlanes; ++lane) {
expected_result[lane] =
binary_op.on_device(first_arg[lane], second_arg[lane]);
}
simd_type const computed_result =
binary_op.on_device(first_arg, second_arg);
device_check_equality(expected_result, computed_result, nlanes);
}
}
template <class Abi, class BinaryOp, class T>
inline void host_check_binary_op_all_loaders(BinaryOp binary_op, std::size_t n,
T const* first_args,
T const* second_args) {
host_check_binary_op_one_loader<Abi, load_element_aligned>(
binary_op, n, first_args, second_args);
host_check_binary_op_one_loader<Abi, load_masked>(binary_op, n, first_args,
second_args);
host_check_binary_op_one_loader<Abi, load_as_scalars>(
binary_op, n, first_args, second_args);
}
template <class Abi, class BinaryOp, class T>
KOKKOS_INLINE_FUNCTION void device_check_binary_op_all_loaders(
BinaryOp binary_op, std::size_t n, T const* first_args,
T const* second_args) {
device_check_binary_op_one_loader<Abi, load_element_aligned>(
binary_op, n, first_args, second_args);
device_check_binary_op_one_loader<Abi, load_masked>(binary_op, n, first_args,
second_args);
device_check_binary_op_one_loader<Abi, load_as_scalars>(
binary_op, n, first_args, second_args);
}
class plus {
public:
template <class T>
auto on_host(T const& a, T const& b) const {
return a + b;
}
template <class T>
KOKKOS_INLINE_FUNCTION auto on_device(T const& a, T const& b) const {
return a + b;
}
};
class minus {
public:
template <class T>
auto on_host(T const& a, T const& b) const {
return a - b;
}
template <class T>
KOKKOS_INLINE_FUNCTION auto on_device(T const& a, T const& b) const {
return a - b;
}
};
class multiplies {
public:
template <class T>
auto on_host(T const& a, T const& b) const {
return a * b;
}
template <class T>
KOKKOS_INLINE_FUNCTION auto on_device(T const& a, T const& b) const {
return a * b;
}
};
class divides {
public:
template <class T>
auto on_host(T const& a, T const& b) const {
return a / b;
}
template <class T>
KOKKOS_INLINE_FUNCTION auto on_device(T const& a, T const& b) const {
return a / b;
}
};
template <class Abi>
inline void host_check_math_ops() {
std::size_t constexpr n = 11;
double const first_args[n] = {1, 2, -1, 10, 0, 1, -2, 10, 0, 1, -2};
double const second_args[n] = {1, 2, 1, 1, 1, -3, -2, 1, 13, -3, -2};
host_check_binary_op_all_loaders<Abi>(plus(), n, first_args, second_args);
host_check_binary_op_all_loaders<Abi>(minus(), n, first_args, second_args);
host_check_binary_op_all_loaders<Abi>(multiplies(), n, first_args,
second_args);
host_check_binary_op_all_loaders<Abi>(divides(), n, first_args, second_args);
}
template <class Abi>
KOKKOS_INLINE_FUNCTION void device_check_math_ops() {
std::size_t constexpr n = 11;
double const first_args[n] = {1, 2, -1, 10, 0, 1, -2, 10, 0, 1, -2};
double const second_args[n] = {1, 2, 1, 1, 1, -3, -2, 1, 13, -3, -2};
device_check_binary_op_all_loaders<Abi>(plus(), n, first_args, second_args);
device_check_binary_op_all_loaders<Abi>(minus(), n, first_args, second_args);
device_check_binary_op_all_loaders<Abi>(multiplies(), n, first_args,
second_args);
device_check_binary_op_all_loaders<Abi>(divides(), n, first_args,
second_args);
}
template <class Abi>
inline void host_check_abi() {
host_check_math_ops<Abi>();
}
template <class Abi>
KOKKOS_INLINE_FUNCTION void device_check_abi() {
device_check_math_ops<Abi>();
}
inline void host_check_abis(Kokkos::Experimental::Impl::abi_set<>) {}
KOKKOS_INLINE_FUNCTION void device_check_abis(
Kokkos::Experimental::Impl::abi_set<>) {}
template <class FirstAbi, class... RestAbis>
inline void host_check_abis(
Kokkos::Experimental::Impl::abi_set<FirstAbi, RestAbis...>) {
host_check_abi<FirstAbi>();
host_check_abis(Kokkos::Experimental::Impl::abi_set<RestAbis...>());
}
template <class FirstAbi, class... RestAbis>
KOKKOS_INLINE_FUNCTION void device_check_abis(
Kokkos::Experimental::Impl::abi_set<FirstAbi, RestAbis...>) {
device_check_abi<FirstAbi>();
device_check_abis(Kokkos::Experimental::Impl::abi_set<RestAbis...>());
}
TEST(simd, host) {
host_check_abis(Kokkos::Experimental::Impl::host_abi_set());
}
class simd_device_functor {
public:
KOKKOS_INLINE_FUNCTION void operator()(int) const {
device_check_abis(Kokkos::Experimental::Impl::device_abi_set());
}
};
TEST(simd, device) {
Kokkos::parallel_for(Kokkos::RangePolicy<Kokkos::IndexType<int>>(0, 1),
simd_device_functor());
}

54
lib/kokkos/simd/unit_tests/UnitTestMain.cpp Normal file
View File

@ -0,0 +1,54 @@
/*
//@HEADER
// ************************************************************************
//
// Kokkos v. 3.0
// Copyright (2020) National Technology & Engineering
// Solutions of Sandia, LLC (NTESS).
//
// Under the terms of Contract DE-NA0003525 with NTESS,
// the U.S. Government retains certain rights in this software.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
// 1. Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
//
// 2. Redistributions in binary form must reproduce the above copyright
// notice, this list of conditions and the following disclaimer in the
// documentation and/or other materials provided with the distribution.
//
// 3. Neither the name of the Corporation nor the names of the
// contributors may be used to endorse or promote products derived from
// this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY
// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE
// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
//
// Questions? Contact Christian R. Trott (crtrott@sandia.gov)
//
// ************************************************************************
//@HEADER
*/
#include <gtest/gtest.h>
#include <Kokkos_Core.hpp>
int main(int argc, char *argv[]) {
Kokkos::initialize(argc, argv);
::testing::InitGoogleTest(&argc, argv);
int result = RUN_ALL_TESTS();
Kokkos::finalize();
return result;
}