mirror of
https://gitlab.com/libeigen/eigen.git
synced 2026-04-10 11:34:33 +08:00
clean up intel packet reductions
This commit is contained in:
Charles Schlosser
committed by
Rasmus Munk Larsen
Rasmus Munk Larsen
parent
a7f183cadb
commit
4fdf87bbf5
@@ -1857,220 +1857,6 @@ EIGEN_STRONG_INLINE void punpackp(Packet4f* vecs) {
|
||||
vecs[0] = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(vecs[0]), 0x00));
|
||||
}
|
||||
|
||||
template <>
|
||||
EIGEN_STRONG_INLINE float predux<Packet4f>(const Packet4f& a) {
|
||||
// Disable SSE3 _mm_hadd_pd that is extremely slow on all existing Intel's architectures
|
||||
// (from Nehalem to Haswell)
|
||||
// #ifdef EIGEN_VECTORIZE_SSE3
|
||||
// Packet4f tmp = _mm_add_ps(a, vec4f_swizzle1(a,2,3,2,3));
|
||||
// return pfirst<Packet4f>(_mm_hadd_ps(tmp, tmp));
|
||||
// #else
|
||||
Packet4f tmp = _mm_add_ps(a, _mm_movehl_ps(a, a));
|
||||
return pfirst<Packet4f>(_mm_add_ss(tmp, _mm_shuffle_ps(tmp, tmp, 1)));
|
||||
// #endif
|
||||
}
|
||||
|
||||
template <>
|
||||
EIGEN_STRONG_INLINE double predux<Packet2d>(const Packet2d& a) {
|
||||
// Disable SSE3 _mm_hadd_pd that is extremely slow on all existing Intel's architectures
|
||||
// (from Nehalem to Haswell)
|
||||
// #ifdef EIGEN_VECTORIZE_SSE3
|
||||
// return pfirst<Packet2d>(_mm_hadd_pd(a, a));
|
||||
// #else
|
||||
return pfirst<Packet2d>(_mm_add_sd(a, _mm_unpackhi_pd(a, a)));
|
||||
// #endif
|
||||
}
|
||||
|
||||
template <>
|
||||
EIGEN_STRONG_INLINE int64_t predux<Packet2l>(const Packet2l& a) {
|
||||
return pfirst<Packet2l>(_mm_add_epi64(a, _mm_unpackhi_epi64(a, a)));
|
||||
}
|
||||
|
||||
#ifdef EIGEN_VECTORIZE_SSSE3
|
||||
template <>
|
||||
EIGEN_STRONG_INLINE int predux<Packet4i>(const Packet4i& a) {
|
||||
Packet4i tmp0 = _mm_hadd_epi32(a, a);
|
||||
return pfirst<Packet4i>(_mm_hadd_epi32(tmp0, tmp0));
|
||||
}
|
||||
template <>
|
||||
EIGEN_STRONG_INLINE uint32_t predux<Packet4ui>(const Packet4ui& a) {
|
||||
Packet4ui tmp0 = _mm_hadd_epi32(a, a);
|
||||
return pfirst<Packet4ui>(_mm_hadd_epi32(tmp0, tmp0));
|
||||
}
|
||||
#else
|
||||
template <>
|
||||
EIGEN_STRONG_INLINE int predux<Packet4i>(const Packet4i& a) {
|
||||
Packet4i tmp = _mm_add_epi32(a, _mm_unpackhi_epi64(a, a));
|
||||
return pfirst(tmp) + pfirst<Packet4i>(_mm_shuffle_epi32(tmp, 1));
|
||||
}
|
||||
template <>
|
||||
EIGEN_STRONG_INLINE uint32_t predux<Packet4ui>(const Packet4ui& a) {
|
||||
Packet4ui tmp = _mm_add_epi32(a, _mm_unpackhi_epi64(a, a));
|
||||
return pfirst(tmp) + pfirst<Packet4ui>(_mm_shuffle_epi32(tmp, 1));
|
||||
}
|
||||
#endif
|
||||
|
||||
template <>
|
||||
EIGEN_STRONG_INLINE bool predux<Packet16b>(const Packet16b& a) {
|
||||
Packet4i tmp = _mm_or_si128(a, _mm_unpackhi_epi64(a, a));
|
||||
return (pfirst(tmp) != 0) || (pfirst<Packet4i>(_mm_shuffle_epi32(tmp, 1)) != 0);
|
||||
}
|
||||
|
||||
// Other reduction functions:
|
||||
|
||||
// mul
|
||||
template <>
|
||||
EIGEN_STRONG_INLINE float predux_mul<Packet4f>(const Packet4f& a) {
|
||||
Packet4f tmp = _mm_mul_ps(a, _mm_movehl_ps(a, a));
|
||||
return pfirst<Packet4f>(_mm_mul_ss(tmp, _mm_shuffle_ps(tmp, tmp, 1)));
|
||||
}
|
||||
template <>
|
||||
EIGEN_STRONG_INLINE double predux_mul<Packet2d>(const Packet2d& a) {
|
||||
return pfirst<Packet2d>(_mm_mul_sd(a, _mm_unpackhi_pd(a, a)));
|
||||
}
|
||||
template <>
|
||||
EIGEN_STRONG_INLINE int64_t predux_mul<Packet2l>(const Packet2l& a) {
|
||||
EIGEN_ALIGN16 int64_t aux[2];
|
||||
pstore(aux, a);
|
||||
return aux[0] * aux[1];
|
||||
}
|
||||
template <>
|
||||
EIGEN_STRONG_INLINE int predux_mul<Packet4i>(const Packet4i& a) {
|
||||
// after some experiments, it is seems this is the fastest way to implement it
|
||||
// for GCC (e.g., reusing pmul is very slow!)
|
||||
// TODO try to call _mm_mul_epu32 directly
|
||||
EIGEN_ALIGN16 int aux[4];
|
||||
pstore(aux, a);
|
||||
return (aux[0] * aux[1]) * (aux[2] * aux[3]);
|
||||
}
|
||||
template <>
|
||||
EIGEN_STRONG_INLINE uint32_t predux_mul<Packet4ui>(const Packet4ui& a) {
|
||||
// after some experiments, it is seems this is the fastest way to implement it
|
||||
// for GCC (eg., reusing pmul is very slow !)
|
||||
// TODO try to call _mm_mul_epu32 directly
|
||||
EIGEN_ALIGN16 uint32_t aux[4];
|
||||
pstore(aux, a);
|
||||
return (aux[0] * aux[1]) * (aux[2] * aux[3]);
|
||||
}
|
||||
|
||||
template <>
|
||||
EIGEN_STRONG_INLINE bool predux_mul<Packet16b>(const Packet16b& a) {
|
||||
Packet4i tmp = _mm_and_si128(a, _mm_unpackhi_epi64(a, a));
|
||||
return ((pfirst<Packet4i>(tmp) == 0x01010101) && (pfirst<Packet4i>(_mm_shuffle_epi32(tmp, 1)) == 0x01010101));
|
||||
}
|
||||
|
||||
// min
|
||||
template <>
|
||||
EIGEN_STRONG_INLINE float predux_min<Packet4f>(const Packet4f& a) {
|
||||
Packet4f tmp = _mm_min_ps(a, _mm_movehl_ps(a, a));
|
||||
return pfirst<Packet4f>(_mm_min_ss(tmp, _mm_shuffle_ps(tmp, tmp, 1)));
|
||||
}
|
||||
template <>
|
||||
EIGEN_STRONG_INLINE double predux_min<Packet2d>(const Packet2d& a) {
|
||||
return pfirst<Packet2d>(_mm_min_sd(a, _mm_unpackhi_pd(a, a)));
|
||||
}
|
||||
template <>
|
||||
EIGEN_STRONG_INLINE int predux_min<Packet4i>(const Packet4i& a) {
|
||||
#ifdef EIGEN_VECTORIZE_SSE4_1
|
||||
Packet4i tmp = _mm_min_epi32(a, _mm_shuffle_epi32(a, _MM_SHUFFLE(0, 0, 3, 2)));
|
||||
return pfirst<Packet4i>(_mm_min_epi32(tmp, _mm_shuffle_epi32(tmp, 1)));
|
||||
#else
|
||||
// after some experiments, it is seems this is the fastest way to implement it
|
||||
// for GCC (eg., it does not like using std::min after the pstore !!)
|
||||
EIGEN_ALIGN16 int aux[4];
|
||||
pstore(aux, a);
|
||||
int aux0 = aux[0] < aux[1] ? aux[0] : aux[1];
|
||||
int aux2 = aux[2] < aux[3] ? aux[2] : aux[3];
|
||||
return aux0 < aux2 ? aux0 : aux2;
|
||||
#endif // EIGEN_VECTORIZE_SSE4_1
|
||||
}
|
||||
template <>
|
||||
EIGEN_STRONG_INLINE uint32_t predux_min<Packet4ui>(const Packet4ui& a) {
|
||||
#ifdef EIGEN_VECTORIZE_SSE4_1
|
||||
Packet4ui tmp = _mm_min_epu32(a, _mm_shuffle_epi32(a, _MM_SHUFFLE(0, 0, 3, 2)));
|
||||
return pfirst<Packet4ui>(_mm_min_epu32(tmp, _mm_shuffle_epi32(tmp, 1)));
|
||||
#else
|
||||
// after some experiments, it is seems this is the fastest way to implement it
|
||||
// for GCC (eg., it does not like using std::min after the pstore !!)
|
||||
EIGEN_ALIGN16 uint32_t aux[4];
|
||||
pstore(aux, a);
|
||||
uint32_t aux0 = aux[0] < aux[1] ? aux[0] : aux[1];
|
||||
uint32_t aux2 = aux[2] < aux[3] ? aux[2] : aux[3];
|
||||
return aux0 < aux2 ? aux0 : aux2;
|
||||
#endif // EIGEN_VECTORIZE_SSE4_1
|
||||
}
|
||||
|
||||
// max
|
||||
template <>
|
||||
EIGEN_STRONG_INLINE float predux_max<Packet4f>(const Packet4f& a) {
|
||||
Packet4f tmp = _mm_max_ps(a, _mm_movehl_ps(a, a));
|
||||
return pfirst<Packet4f>(_mm_max_ss(tmp, _mm_shuffle_ps(tmp, tmp, 1)));
|
||||
}
|
||||
template <>
|
||||
EIGEN_STRONG_INLINE double predux_max<Packet2d>(const Packet2d& a) {
|
||||
return pfirst<Packet2d>(_mm_max_sd(a, _mm_unpackhi_pd(a, a)));
|
||||
}
|
||||
template <>
|
||||
EIGEN_STRONG_INLINE int predux_max<Packet4i>(const Packet4i& a) {
|
||||
#ifdef EIGEN_VECTORIZE_SSE4_1
|
||||
Packet4i tmp = _mm_max_epi32(a, _mm_shuffle_epi32(a, _MM_SHUFFLE(0, 0, 3, 2)));
|
||||
return pfirst<Packet4i>(_mm_max_epi32(tmp, _mm_shuffle_epi32(tmp, 1)));
|
||||
#else
|
||||
// after some experiments, it is seems this is the fastest way to implement it
|
||||
// for GCC (eg., it does not like using std::min after the pstore !!)
|
||||
EIGEN_ALIGN16 int aux[4];
|
||||
pstore(aux, a);
|
||||
int aux0 = aux[0] > aux[1] ? aux[0] : aux[1];
|
||||
int aux2 = aux[2] > aux[3] ? aux[2] : aux[3];
|
||||
return aux0 > aux2 ? aux0 : aux2;
|
||||
#endif // EIGEN_VECTORIZE_SSE4_1
|
||||
}
|
||||
template <>
|
||||
EIGEN_STRONG_INLINE uint32_t predux_max<Packet4ui>(const Packet4ui& a) {
|
||||
#ifdef EIGEN_VECTORIZE_SSE4_1
|
||||
Packet4ui tmp = _mm_max_epu32(a, _mm_shuffle_epi32(a, _MM_SHUFFLE(0, 0, 3, 2)));
|
||||
return pfirst<Packet4ui>(_mm_max_epu32(tmp, _mm_shuffle_epi32(tmp, 1)));
|
||||
#else
|
||||
// after some experiments, it is seems this is the fastest way to implement it
|
||||
// for GCC (eg., it does not like using std::min after the pstore !!)
|
||||
EIGEN_ALIGN16 uint32_t aux[4];
|
||||
pstore(aux, a);
|
||||
uint32_t aux0 = aux[0] > aux[1] ? aux[0] : aux[1];
|
||||
uint32_t aux2 = aux[2] > aux[3] ? aux[2] : aux[3];
|
||||
return aux0 > aux2 ? aux0 : aux2;
|
||||
#endif // EIGEN_VECTORIZE_SSE4_1
|
||||
}
|
||||
|
||||
// not needed yet
|
||||
// template<> EIGEN_STRONG_INLINE bool predux_all(const Packet4f& x)
|
||||
// {
|
||||
// return _mm_movemask_ps(x) == 0xF;
|
||||
// }
|
||||
|
||||
template <>
|
||||
EIGEN_STRONG_INLINE bool predux_any(const Packet2d& x) {
|
||||
return _mm_movemask_pd(x) != 0x0;
|
||||
}
|
||||
|
||||
template <>
|
||||
EIGEN_STRONG_INLINE bool predux_any(const Packet4f& x) {
|
||||
return _mm_movemask_ps(x) != 0x0;
|
||||
}
|
||||
|
||||
template <>
|
||||
EIGEN_STRONG_INLINE bool predux_any(const Packet2l& x) {
|
||||
return _mm_movemask_pd(_mm_castsi128_pd(x)) != 0x0;
|
||||
}
|
||||
|
||||
template <>
|
||||
EIGEN_STRONG_INLINE bool predux_any(const Packet4i& x) {
|
||||
return _mm_movemask_ps(_mm_castsi128_ps(x)) != 0x0;
|
||||
}
|
||||
template <>
|
||||
EIGEN_STRONG_INLINE bool predux_any(const Packet4ui& x) {
|
||||
return _mm_movemask_ps(_mm_castsi128_ps(x)) != 0x0;
|
||||
}
|
||||
|
||||
EIGEN_STRONG_INLINE void ptranspose(PacketBlock<Packet4f, 4>& kernel) {
|
||||
_MM_TRANSPOSE4_PS(kernel.packet[0], kernel.packet[1], kernel.packet[2], kernel.packet[3]);
|
||||
}
|
||||
|
||||
329
Eigen/src/Core/arch/SSE/Reductions.h
Normal file
329
Eigen/src/Core/arch/SSE/Reductions.h
Normal file
@@ -0,0 +1,329 @@
|
||||
// This file is part of Eigen, a lightweight C++ template library
|
||||
// for linear algebra.
|
||||
//
|
||||
// Copyright (C) 2025 Charlie Schlosser <cs.schlosser@gmail.com>
|
||||
//
|
||||
// This Source Code Form is subject to the terms of the Mozilla
|
||||
// Public License v. 2.0. If a copy of the MPL was not distributed
|
||||
// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
|
||||
#ifndef EIGEN_REDUCTIONS_SSE_H
|
||||
#define EIGEN_REDUCTIONS_SSE_H
|
||||
|
||||
// IWYU pragma: private
|
||||
#include "../../InternalHeaderCheck.h"
|
||||
|
||||
namespace Eigen {
|
||||
|
||||
namespace internal {
|
||||
|
||||
template <typename Packet>
|
||||
struct sse_add_wrapper {
|
||||
static EIGEN_STRONG_INLINE Packet packetOp(const Packet& a, const Packet& b) { return padd<Packet>(a, b); }
|
||||
};
|
||||
|
||||
template <typename Packet>
|
||||
struct sse_mul_wrapper {
|
||||
static EIGEN_STRONG_INLINE Packet packetOp(const Packet& a, const Packet& b) { return pmul<Packet>(a, b); }
|
||||
};
|
||||
|
||||
template <typename Packet>
|
||||
struct sse_min_wrapper {
|
||||
static EIGEN_STRONG_INLINE Packet packetOp(const Packet& a, const Packet& b) { return pmin<Packet>(a, b); }
|
||||
};
|
||||
|
||||
template <int NaNPropagation, typename Packet>
|
||||
struct sse_min_prop_wrapper {
|
||||
static EIGEN_STRONG_INLINE Packet packetOp(const Packet& a, const Packet& b) {
|
||||
return pmin<NaNPropagation, Packet>(a, b);
|
||||
}
|
||||
};
|
||||
|
||||
template <typename Packet>
|
||||
struct sse_max_wrapper {
|
||||
static EIGEN_STRONG_INLINE Packet packetOp(const Packet& a, const Packet& b) { return pmax<Packet>(a, b); }
|
||||
};
|
||||
|
||||
template <int NaNPropagation, typename Packet>
|
||||
struct sse_max_prop_wrapper {
|
||||
static EIGEN_STRONG_INLINE Packet packetOp(const Packet& a, const Packet& b) {
|
||||
return pmax<NaNPropagation, Packet>(a, b);
|
||||
}
|
||||
};
|
||||
|
||||
template <typename Packet, typename Op>
|
||||
struct sse_predux_common;
|
||||
|
||||
template <typename Packet>
|
||||
struct sse_predux_impl : sse_predux_common<Packet, sse_add_wrapper<Packet>> {};
|
||||
|
||||
template <typename Packet>
|
||||
struct sse_predux_mul_impl : sse_predux_common<Packet, sse_mul_wrapper<Packet>> {};
|
||||
|
||||
template <typename Packet>
|
||||
struct sse_predux_min_impl : sse_predux_common<Packet, sse_min_wrapper<Packet>> {};
|
||||
|
||||
template <int NaNPropagation, typename Packet>
|
||||
struct sse_predux_min_prop_impl : sse_predux_common<Packet, sse_min_prop_wrapper<NaNPropagation, Packet>> {};
|
||||
|
||||
template <typename Packet>
|
||||
struct sse_predux_max_impl : sse_predux_common<Packet, sse_max_wrapper<Packet>> {};
|
||||
|
||||
template <int NaNPropagation, typename Packet>
|
||||
struct sse_predux_max_prop_impl : sse_predux_common<Packet, sse_max_prop_wrapper<NaNPropagation, Packet>> {};
|
||||
|
||||
/* -- -- -- -- -- -- -- -- -- -- -- -- Packet16b -- -- -- -- -- -- -- -- -- -- -- -- */
|
||||
|
||||
template <>
|
||||
EIGEN_STRONG_INLINE bool predux(const Packet16b& a) {
|
||||
Packet4i tmp = _mm_or_si128(a, _mm_unpackhi_epi64(a, a));
|
||||
return (pfirst(tmp) != 0) || (pfirst<Packet4i>(_mm_shuffle_epi32(tmp, 1)) != 0);
|
||||
}
|
||||
|
||||
template <>
|
||||
EIGEN_STRONG_INLINE bool predux_mul(const Packet16b& a) {
|
||||
Packet4i tmp = _mm_and_si128(a, _mm_unpackhi_epi64(a, a));
|
||||
return ((pfirst<Packet4i>(tmp) == 0x01010101) && (pfirst<Packet4i>(_mm_shuffle_epi32(tmp, 1)) == 0x01010101));
|
||||
}
|
||||
|
||||
/* -- -- -- -- -- -- -- -- -- -- -- -- Packet4i -- -- -- -- -- -- -- -- -- -- -- -- */
|
||||
|
||||
template <typename Op>
|
||||
struct sse_predux_common<Packet4i, Op> {
|
||||
static EIGEN_STRONG_INLINE int run(const Packet4i& a) {
|
||||
Packet4i tmp;
|
||||
tmp = Op::packetOp(a, _mm_shuffle_epi32(a, _MM_SHUFFLE(0, 1, 2, 3)));
|
||||
tmp = Op::packetOp(tmp, _mm_unpackhi_epi32(tmp, tmp));
|
||||
return _mm_cvtsi128_si32(tmp);
|
||||
}
|
||||
};
|
||||
|
||||
template <>
|
||||
EIGEN_STRONG_INLINE int predux(const Packet4i& a) {
|
||||
return sse_predux_impl<Packet4i>::run(a);
|
||||
}
|
||||
|
||||
template <>
|
||||
EIGEN_STRONG_INLINE int predux_mul(const Packet4i& a) {
|
||||
return sse_predux_mul_impl<Packet4i>::run(a);
|
||||
}
|
||||
|
||||
#ifdef EIGEN_VECTORIZE_SSE4_1
|
||||
template <>
|
||||
EIGEN_STRONG_INLINE int predux_min(const Packet4i& a) {
|
||||
return sse_predux_min_impl<Packet4i>::run(a);
|
||||
}
|
||||
|
||||
template <>
|
||||
EIGEN_STRONG_INLINE int predux_max(const Packet4i& a) {
|
||||
return sse_predux_max_impl<Packet4i>::run(a);
|
||||
}
|
||||
#endif
|
||||
|
||||
template <>
|
||||
EIGEN_STRONG_INLINE bool predux_any(const Packet4i& x) {
|
||||
return _mm_movemask_ps(_mm_castsi128_ps(x)) != 0x0;
|
||||
}
|
||||
|
||||
/* -- -- -- -- -- -- -- -- -- -- -- -- Packet4ui -- -- -- -- -- -- -- -- -- -- -- -- */
|
||||
|
||||
template <typename Op>
|
||||
struct sse_predux_common<Packet4ui, Op> {
|
||||
static EIGEN_STRONG_INLINE uint32_t run(const Packet4ui& a) {
|
||||
Packet4ui tmp;
|
||||
tmp = Op::packetOp(a, _mm_shuffle_epi32(a, _MM_SHUFFLE(0, 1, 2, 3)));
|
||||
tmp = Op::packetOp(tmp, _mm_unpackhi_epi32(tmp, tmp));
|
||||
return static_cast<uint32_t>(_mm_cvtsi128_si32(tmp));
|
||||
}
|
||||
};
|
||||
|
||||
template <>
|
||||
EIGEN_STRONG_INLINE uint32_t predux(const Packet4ui& a) {
|
||||
return sse_predux_impl<Packet4ui>::run(a);
|
||||
}
|
||||
|
||||
template <>
|
||||
EIGEN_STRONG_INLINE uint32_t predux_mul(const Packet4ui& a) {
|
||||
return sse_predux_mul_impl<Packet4ui>::run(a);
|
||||
}
|
||||
|
||||
#ifdef EIGEN_VECTORIZE_SSE4_1
|
||||
template <>
|
||||
EIGEN_STRONG_INLINE uint32_t predux_min(const Packet4ui& a) {
|
||||
return sse_predux_min_impl<Packet4ui>::run(a);
|
||||
}
|
||||
|
||||
template <>
|
||||
EIGEN_STRONG_INLINE uint32_t predux_max(const Packet4ui& a) {
|
||||
return sse_predux_max_impl<Packet4ui>::run(a);
|
||||
}
|
||||
#endif
|
||||
|
||||
template <>
|
||||
EIGEN_STRONG_INLINE bool predux_any(const Packet4ui& x) {
|
||||
return _mm_movemask_ps(_mm_castsi128_ps(x)) != 0x0;
|
||||
}
|
||||
|
||||
/* -- -- -- -- -- -- -- -- -- -- -- -- Packet2l -- -- -- -- -- -- -- -- -- -- -- -- */
|
||||
|
||||
template <typename Op>
|
||||
struct sse_predux_common<Packet2l, Op> {
|
||||
static EIGEN_STRONG_INLINE int64_t run(const Packet2l& a) {
|
||||
Packet2l tmp;
|
||||
tmp = Op::packetOp(a, _mm_unpackhi_epi64(a, a));
|
||||
return pfirst(tmp);
|
||||
}
|
||||
};
|
||||
|
||||
template <>
|
||||
EIGEN_STRONG_INLINE int64_t predux(const Packet2l& a) {
|
||||
return sse_predux_impl<Packet2l>::run(a);
|
||||
}
|
||||
|
||||
template <>
|
||||
EIGEN_STRONG_INLINE bool predux_any(const Packet2l& x) {
|
||||
return _mm_movemask_pd(_mm_castsi128_pd(x)) != 0x0;
|
||||
}
|
||||
|
||||
/* -- -- -- -- -- -- -- -- -- -- -- -- Packet4f -- -- -- -- -- -- -- -- -- -- -- -- */
|
||||
|
||||
template <typename Op>
|
||||
struct sse_predux_common<Packet4f, Op> {
|
||||
static EIGEN_STRONG_INLINE float run(const Packet4f& a) {
|
||||
Packet4f tmp;
|
||||
tmp = Op::packetOp(a, _mm_movehl_ps(a, a));
|
||||
#ifdef EIGEN_VECTORIZE_SSE3
|
||||
tmp = Op::packetOp(tmp, _mm_movehdup_ps(tmp));
|
||||
#else
|
||||
tmp = Op::packetOp(tmp, _mm_shuffle_ps(tmp, tmp, 1));
|
||||
#endif
|
||||
return _mm_cvtss_f32(tmp);
|
||||
}
|
||||
};
|
||||
|
||||
template <>
|
||||
EIGEN_STRONG_INLINE float predux(const Packet4f& a) {
|
||||
return sse_predux_impl<Packet4f>::run(a);
|
||||
}
|
||||
|
||||
template <>
|
||||
EIGEN_STRONG_INLINE float predux_mul(const Packet4f& a) {
|
||||
return sse_predux_mul_impl<Packet4f>::run(a);
|
||||
}
|
||||
|
||||
template <>
|
||||
EIGEN_STRONG_INLINE float predux_min(const Packet4f& a) {
|
||||
return sse_predux_min_impl<Packet4f>::run(a);
|
||||
}
|
||||
|
||||
template <>
|
||||
EIGEN_STRONG_INLINE float predux_min<PropagateFast>(const Packet4f& a) {
|
||||
return sse_predux_min_prop_impl<PropagateFast, Packet4f>::run(a);
|
||||
}
|
||||
|
||||
template <>
|
||||
EIGEN_STRONG_INLINE float predux_min<PropagateNumbers>(const Packet4f& a) {
|
||||
return sse_predux_min_prop_impl<PropagateNumbers, Packet4f>::run(a);
|
||||
}
|
||||
|
||||
template <>
|
||||
EIGEN_STRONG_INLINE float predux_min<PropagateNaN>(const Packet4f& a) {
|
||||
return sse_predux_min_prop_impl<PropagateNaN, Packet4f>::run(a);
|
||||
}
|
||||
|
||||
template <>
|
||||
EIGEN_STRONG_INLINE float predux_max(const Packet4f& a) {
|
||||
return sse_predux_max_impl<Packet4f>::run(a);
|
||||
}
|
||||
|
||||
template <>
|
||||
EIGEN_STRONG_INLINE float predux_max<PropagateFast>(const Packet4f& a) {
|
||||
return sse_predux_max_prop_impl<PropagateFast, Packet4f>::run(a);
|
||||
}
|
||||
|
||||
template <>
|
||||
EIGEN_STRONG_INLINE float predux_max<PropagateNumbers>(const Packet4f& a) {
|
||||
return sse_predux_max_prop_impl<PropagateNumbers, Packet4f>::run(a);
|
||||
}
|
||||
|
||||
template <>
|
||||
EIGEN_STRONG_INLINE float predux_max<PropagateNaN>(const Packet4f& a) {
|
||||
return sse_predux_max_prop_impl<PropagateNaN, Packet4f>::run(a);
|
||||
}
|
||||
|
||||
template <>
|
||||
EIGEN_STRONG_INLINE bool predux_any(const Packet4f& x) {
|
||||
return _mm_movemask_ps(x) != 0x0;
|
||||
}
|
||||
|
||||
/* -- -- -- -- -- -- -- -- -- -- -- -- Packet2d -- -- -- -- -- -- -- -- -- -- -- -- */
|
||||
|
||||
template <typename Op>
|
||||
struct sse_predux_common<Packet2d, Op> {
|
||||
static EIGEN_STRONG_INLINE double run(const Packet2d& a) {
|
||||
Packet2d tmp;
|
||||
tmp = Op::packetOp(a, _mm_unpackhi_pd(a, a));
|
||||
return _mm_cvtsd_f64(tmp);
|
||||
}
|
||||
};
|
||||
|
||||
template <>
|
||||
EIGEN_STRONG_INLINE double predux(const Packet2d& a) {
|
||||
return sse_predux_impl<Packet2d>::run(a);
|
||||
}
|
||||
|
||||
template <>
|
||||
EIGEN_STRONG_INLINE double predux_mul(const Packet2d& a) {
|
||||
return sse_predux_mul_impl<Packet2d>::run(a);
|
||||
}
|
||||
|
||||
template <>
|
||||
EIGEN_STRONG_INLINE double predux_min(const Packet2d& a) {
|
||||
return sse_predux_min_impl<Packet2d>::run(a);
|
||||
}
|
||||
|
||||
template <>
|
||||
EIGEN_STRONG_INLINE double predux_min<PropagateFast>(const Packet2d& a) {
|
||||
return sse_predux_min_prop_impl<PropagateFast, Packet2d>::run(a);
|
||||
}
|
||||
|
||||
template <>
|
||||
EIGEN_STRONG_INLINE double predux_min<PropagateNumbers>(const Packet2d& a) {
|
||||
return sse_predux_min_prop_impl<PropagateNumbers, Packet2d>::run(a);
|
||||
}
|
||||
|
||||
template <>
|
||||
EIGEN_STRONG_INLINE double predux_min<PropagateNaN>(const Packet2d& a) {
|
||||
return sse_predux_min_prop_impl<PropagateNaN, Packet2d>::run(a);
|
||||
}
|
||||
|
||||
template <>
|
||||
EIGEN_STRONG_INLINE double predux_max(const Packet2d& a) {
|
||||
return sse_predux_max_impl<Packet2d>::run(a);
|
||||
}
|
||||
|
||||
template <>
|
||||
EIGEN_STRONG_INLINE double predux_max<PropagateFast>(const Packet2d& a) {
|
||||
return sse_predux_max_prop_impl<PropagateFast, Packet2d>::run(a);
|
||||
}
|
||||
|
||||
template <>
|
||||
EIGEN_STRONG_INLINE double predux_max<PropagateNumbers>(const Packet2d& a) {
|
||||
return sse_predux_max_prop_impl<PropagateNumbers, Packet2d>::run(a);
|
||||
}
|
||||
|
||||
template <>
|
||||
EIGEN_STRONG_INLINE double predux_max<PropagateNaN>(const Packet2d& a) {
|
||||
return sse_predux_max_prop_impl<PropagateNaN, Packet2d>::run(a);
|
||||
}
|
||||
|
||||
template <>
|
||||
EIGEN_STRONG_INLINE bool predux_any(const Packet2d& x) {
|
||||
return _mm_movemask_pd(x) != 0x0;
|
||||
}
|
||||
|
||||
} // end namespace internal
|
||||
|
||||
} // end namespace Eigen
|
||||
|
||||
#endif // EIGEN_REDUCTIONS_SSE_H
|
||||
Reference in New Issue
Block a user