mirror of
https://gitlab.com/libeigen/eigen.git
synced 2026-04-10 11:34:33 +08:00
bug #86 : use internal:: namespace instead of ei_ prefix
This commit is contained in:
Benoit Jacob
@@ -25,6 +25,8 @@
|
||||
#ifndef EIGEN_COMPLEX_SSE_H
|
||||
#define EIGEN_COMPLEX_SSE_H
|
||||
|
||||
namespace internal {
|
||||
|
||||
//---------- float ----------
|
||||
struct Packet2cf
|
||||
{
|
||||
@@ -33,7 +35,7 @@ struct Packet2cf
|
||||
__m128 v;
|
||||
};
|
||||
|
||||
template<> struct ei_packet_traits<std::complex<float> > : ei_default_packet_traits
|
||||
template<> struct packet_traits<std::complex<float> > : default_packet_traits
|
||||
{
|
||||
typedef Packet2cf type;
|
||||
enum {
|
||||
@@ -54,85 +56,85 @@ template<> struct ei_packet_traits<std::complex<float> > : ei_default_packet_tr
|
||||
};
|
||||
};
|
||||
|
||||
template<> struct ei_unpacket_traits<Packet2cf> { typedef std::complex<float> type; enum {size=2}; };
|
||||
template<> struct unpacket_traits<Packet2cf> { typedef std::complex<float> type; enum {size=2}; };
|
||||
|
||||
template<> EIGEN_STRONG_INLINE Packet2cf ei_padd<Packet2cf>(const Packet2cf& a, const Packet2cf& b) { return Packet2cf(_mm_add_ps(a.v,b.v)); }
|
||||
template<> EIGEN_STRONG_INLINE Packet2cf ei_psub<Packet2cf>(const Packet2cf& a, const Packet2cf& b) { return Packet2cf(_mm_sub_ps(a.v,b.v)); }
|
||||
template<> EIGEN_STRONG_INLINE Packet2cf ei_pnegate(const Packet2cf& a)
|
||||
template<> EIGEN_STRONG_INLINE Packet2cf padd<Packet2cf>(const Packet2cf& a, const Packet2cf& b) { return Packet2cf(_mm_add_ps(a.v,b.v)); }
|
||||
template<> EIGEN_STRONG_INLINE Packet2cf psub<Packet2cf>(const Packet2cf& a, const Packet2cf& b) { return Packet2cf(_mm_sub_ps(a.v,b.v)); }
|
||||
template<> EIGEN_STRONG_INLINE Packet2cf pnegate(const Packet2cf& a)
|
||||
{
|
||||
const __m128 mask = _mm_castsi128_ps(_mm_setr_epi32(0x80000000,0x80000000,0x80000000,0x80000000));
|
||||
return Packet2cf(_mm_xor_ps(a.v,mask));
|
||||
}
|
||||
template<> EIGEN_STRONG_INLINE Packet2cf ei_pconj(const Packet2cf& a)
|
||||
template<> EIGEN_STRONG_INLINE Packet2cf pconj(const Packet2cf& a)
|
||||
{
|
||||
const __m128 mask = _mm_castsi128_ps(_mm_setr_epi32(0x00000000,0x80000000,0x00000000,0x80000000));
|
||||
return Packet2cf(_mm_xor_ps(a.v,mask));
|
||||
}
|
||||
|
||||
template<> EIGEN_STRONG_INLINE Packet2cf ei_pmul<Packet2cf>(const Packet2cf& a, const Packet2cf& b)
|
||||
template<> EIGEN_STRONG_INLINE Packet2cf pmul<Packet2cf>(const Packet2cf& a, const Packet2cf& b)
|
||||
{
|
||||
// TODO optimize it for SSE3 and 4
|
||||
#ifdef EIGEN_VECTORIZE_SSE3
|
||||
return Packet2cf(_mm_addsub_ps(_mm_mul_ps(_mm_moveldup_ps(a.v), b.v),
|
||||
_mm_mul_ps(_mm_movehdup_ps(a.v),
|
||||
ei_vec4f_swizzle1(b.v, 1, 0, 3, 2))));
|
||||
// return Packet2cf(_mm_addsub_ps(_mm_mul_ps(ei_vec4f_swizzle1(a.v, 0, 0, 2, 2), b.v),
|
||||
// _mm_mul_ps(ei_vec4f_swizzle1(a.v, 1, 1, 3, 3),
|
||||
// ei_vec4f_swizzle1(b.v, 1, 0, 3, 2))));
|
||||
vec4f_swizzle1(b.v, 1, 0, 3, 2))));
|
||||
// return Packet2cf(_mm_addsub_ps(_mm_mul_ps(vec4f_swizzle1(a.v, 0, 0, 2, 2), b.v),
|
||||
// _mm_mul_ps(vec4f_swizzle1(a.v, 1, 1, 3, 3),
|
||||
// vec4f_swizzle1(b.v, 1, 0, 3, 2))));
|
||||
#else
|
||||
const __m128 mask = _mm_castsi128_ps(_mm_setr_epi32(0x80000000,0x00000000,0x80000000,0x00000000));
|
||||
return Packet2cf(_mm_add_ps(_mm_mul_ps(ei_vec4f_swizzle1(a.v, 0, 0, 2, 2), b.v),
|
||||
_mm_xor_ps(_mm_mul_ps(ei_vec4f_swizzle1(a.v, 1, 1, 3, 3),
|
||||
ei_vec4f_swizzle1(b.v, 1, 0, 3, 2)), mask)));
|
||||
return Packet2cf(_mm_add_ps(_mm_mul_ps(vec4f_swizzle1(a.v, 0, 0, 2, 2), b.v),
|
||||
_mm_xor_ps(_mm_mul_ps(vec4f_swizzle1(a.v, 1, 1, 3, 3),
|
||||
vec4f_swizzle1(b.v, 1, 0, 3, 2)), mask)));
|
||||
#endif
|
||||
}
|
||||
|
||||
template<> EIGEN_STRONG_INLINE Packet2cf ei_pand <Packet2cf>(const Packet2cf& a, const Packet2cf& b) { return Packet2cf(_mm_and_ps(a.v,b.v)); }
|
||||
template<> EIGEN_STRONG_INLINE Packet2cf ei_por <Packet2cf>(const Packet2cf& a, const Packet2cf& b) { return Packet2cf(_mm_or_ps(a.v,b.v)); }
|
||||
template<> EIGEN_STRONG_INLINE Packet2cf ei_pxor <Packet2cf>(const Packet2cf& a, const Packet2cf& b) { return Packet2cf(_mm_xor_ps(a.v,b.v)); }
|
||||
template<> EIGEN_STRONG_INLINE Packet2cf ei_pandnot<Packet2cf>(const Packet2cf& a, const Packet2cf& b) { return Packet2cf(_mm_andnot_ps(a.v,b.v)); }
|
||||
template<> EIGEN_STRONG_INLINE Packet2cf pand <Packet2cf>(const Packet2cf& a, const Packet2cf& b) { return Packet2cf(_mm_and_ps(a.v,b.v)); }
|
||||
template<> EIGEN_STRONG_INLINE Packet2cf por <Packet2cf>(const Packet2cf& a, const Packet2cf& b) { return Packet2cf(_mm_or_ps(a.v,b.v)); }
|
||||
template<> EIGEN_STRONG_INLINE Packet2cf pxor <Packet2cf>(const Packet2cf& a, const Packet2cf& b) { return Packet2cf(_mm_xor_ps(a.v,b.v)); }
|
||||
template<> EIGEN_STRONG_INLINE Packet2cf pandnot<Packet2cf>(const Packet2cf& a, const Packet2cf& b) { return Packet2cf(_mm_andnot_ps(a.v,b.v)); }
|
||||
|
||||
template<> EIGEN_STRONG_INLINE Packet2cf ei_pload <Packet2cf>(const std::complex<float>* from) { EIGEN_DEBUG_ALIGNED_LOAD return Packet2cf(ei_pload<Packet4f>(&ei_real_ref(*from))); }
|
||||
template<> EIGEN_STRONG_INLINE Packet2cf ei_ploadu<Packet2cf>(const std::complex<float>* from) { EIGEN_DEBUG_UNALIGNED_LOAD return Packet2cf(ei_ploadu<Packet4f>(&ei_real_ref(*from))); }
|
||||
template<> EIGEN_STRONG_INLINE Packet2cf pload <Packet2cf>(const std::complex<float>* from) { EIGEN_DEBUG_ALIGNED_LOAD return Packet2cf(pload<Packet4f>(&real_ref(*from))); }
|
||||
template<> EIGEN_STRONG_INLINE Packet2cf ploadu<Packet2cf>(const std::complex<float>* from) { EIGEN_DEBUG_UNALIGNED_LOAD return Packet2cf(ploadu<Packet4f>(&real_ref(*from))); }
|
||||
|
||||
template<> EIGEN_STRONG_INLINE void ei_pstore <std::complex<float> >(std::complex<float> * to, const Packet2cf& from) { EIGEN_DEBUG_ALIGNED_STORE ei_pstore(&ei_real_ref(*to), from.v); }
|
||||
template<> EIGEN_STRONG_INLINE void ei_pstoreu<std::complex<float> >(std::complex<float> * to, const Packet2cf& from) { EIGEN_DEBUG_UNALIGNED_STORE ei_pstoreu(&ei_real_ref(*to), from.v); }
|
||||
template<> EIGEN_STRONG_INLINE void pstore <std::complex<float> >(std::complex<float> * to, const Packet2cf& from) { EIGEN_DEBUG_ALIGNED_STORE pstore(&real_ref(*to), from.v); }
|
||||
template<> EIGEN_STRONG_INLINE void pstoreu<std::complex<float> >(std::complex<float> * to, const Packet2cf& from) { EIGEN_DEBUG_UNALIGNED_STORE pstoreu(&real_ref(*to), from.v); }
|
||||
|
||||
template<> EIGEN_STRONG_INLINE void ei_prefetch<std::complex<float> >(const std::complex<float> * addr) { _mm_prefetch((const char*)(addr), _MM_HINT_T0); }
|
||||
template<> EIGEN_STRONG_INLINE void prefetch<std::complex<float> >(const std::complex<float> * addr) { _mm_prefetch((const char*)(addr), _MM_HINT_T0); }
|
||||
|
||||
template<> EIGEN_STRONG_INLINE Packet2cf ei_pset1<Packet2cf>(const std::complex<float>& from)
|
||||
template<> EIGEN_STRONG_INLINE Packet2cf pset1<Packet2cf>(const std::complex<float>& from)
|
||||
{
|
||||
Packet2cf res;
|
||||
res.v = _mm_loadl_pi(res.v, (const __m64*)&from);
|
||||
return Packet2cf(_mm_movelh_ps(res.v,res.v));
|
||||
}
|
||||
|
||||
template<> EIGEN_STRONG_INLINE std::complex<float> ei_pfirst<Packet2cf>(const Packet2cf& a)
|
||||
template<> EIGEN_STRONG_INLINE std::complex<float> pfirst<Packet2cf>(const Packet2cf& a)
|
||||
{
|
||||
std::complex<float> res;
|
||||
_mm_storel_pi((__m64*)&res, a.v);
|
||||
return res;
|
||||
}
|
||||
|
||||
template<> EIGEN_STRONG_INLINE Packet2cf ei_preverse(const Packet2cf& a) { return Packet2cf(_mm_castpd_ps(ei_preverse(_mm_castps_pd(a.v)))); }
|
||||
template<> EIGEN_STRONG_INLINE Packet2cf preverse(const Packet2cf& a) { return Packet2cf(_mm_castpd_ps(preverse(_mm_castps_pd(a.v)))); }
|
||||
|
||||
template<> EIGEN_STRONG_INLINE std::complex<float> ei_predux<Packet2cf>(const Packet2cf& a)
|
||||
template<> EIGEN_STRONG_INLINE std::complex<float> predux<Packet2cf>(const Packet2cf& a)
|
||||
{
|
||||
return ei_pfirst(Packet2cf(_mm_add_ps(a.v, _mm_movehl_ps(a.v,a.v))));
|
||||
return pfirst(Packet2cf(_mm_add_ps(a.v, _mm_movehl_ps(a.v,a.v))));
|
||||
}
|
||||
|
||||
template<> EIGEN_STRONG_INLINE Packet2cf ei_preduxp<Packet2cf>(const Packet2cf* vecs)
|
||||
template<> EIGEN_STRONG_INLINE Packet2cf preduxp<Packet2cf>(const Packet2cf* vecs)
|
||||
{
|
||||
return Packet2cf(_mm_add_ps(_mm_movelh_ps(vecs[0].v,vecs[1].v), _mm_movehl_ps(vecs[1].v,vecs[0].v)));
|
||||
}
|
||||
|
||||
template<> EIGEN_STRONG_INLINE std::complex<float> ei_predux_mul<Packet2cf>(const Packet2cf& a)
|
||||
template<> EIGEN_STRONG_INLINE std::complex<float> predux_mul<Packet2cf>(const Packet2cf& a)
|
||||
{
|
||||
return ei_pfirst(ei_pmul(a, Packet2cf(_mm_movehl_ps(a.v,a.v))));
|
||||
return pfirst(pmul(a, Packet2cf(_mm_movehl_ps(a.v,a.v))));
|
||||
}
|
||||
|
||||
template<int Offset>
|
||||
struct ei_palign_impl<Offset,Packet2cf>
|
||||
struct palign_impl<Offset,Packet2cf>
|
||||
{
|
||||
EIGEN_STRONG_INLINE static void run(Packet2cf& first, const Packet2cf& second)
|
||||
{
|
||||
@@ -144,89 +146,89 @@ struct ei_palign_impl<Offset,Packet2cf>
|
||||
}
|
||||
};
|
||||
|
||||
template<> struct ei_conj_helper<Packet2cf, Packet2cf, false,true>
|
||||
template<> struct conj_helper<Packet2cf, Packet2cf, false,true>
|
||||
{
|
||||
EIGEN_STRONG_INLINE Packet2cf pmadd(const Packet2cf& x, const Packet2cf& y, const Packet2cf& c) const
|
||||
{ return ei_padd(pmul(x,y),c); }
|
||||
{ return padd(pmul(x,y),c); }
|
||||
|
||||
EIGEN_STRONG_INLINE Packet2cf pmul(const Packet2cf& a, const Packet2cf& b) const
|
||||
{
|
||||
#ifdef EIGEN_VECTORIZE_SSE3
|
||||
return ei_pmul(a, ei_pconj(b));
|
||||
return pmul(a, pconj(b));
|
||||
#else
|
||||
const __m128 mask = _mm_castsi128_ps(_mm_setr_epi32(0x00000000,0x80000000,0x00000000,0x80000000));
|
||||
return Packet2cf(_mm_add_ps(_mm_xor_ps(_mm_mul_ps(ei_vec4f_swizzle1(a.v, 0, 0, 2, 2), b.v), mask),
|
||||
_mm_mul_ps(ei_vec4f_swizzle1(a.v, 1, 1, 3, 3),
|
||||
ei_vec4f_swizzle1(b.v, 1, 0, 3, 2))));
|
||||
return Packet2cf(_mm_add_ps(_mm_xor_ps(_mm_mul_ps(vec4f_swizzle1(a.v, 0, 0, 2, 2), b.v), mask),
|
||||
_mm_mul_ps(vec4f_swizzle1(a.v, 1, 1, 3, 3),
|
||||
vec4f_swizzle1(b.v, 1, 0, 3, 2))));
|
||||
#endif
|
||||
}
|
||||
};
|
||||
|
||||
template<> struct ei_conj_helper<Packet2cf, Packet2cf, true,false>
|
||||
template<> struct conj_helper<Packet2cf, Packet2cf, true,false>
|
||||
{
|
||||
EIGEN_STRONG_INLINE Packet2cf pmadd(const Packet2cf& x, const Packet2cf& y, const Packet2cf& c) const
|
||||
{ return ei_padd(pmul(x,y),c); }
|
||||
{ return padd(pmul(x,y),c); }
|
||||
|
||||
EIGEN_STRONG_INLINE Packet2cf pmul(const Packet2cf& a, const Packet2cf& b) const
|
||||
{
|
||||
#ifdef EIGEN_VECTORIZE_SSE3
|
||||
return ei_pmul(ei_pconj(a), b);
|
||||
return pmul(pconj(a), b);
|
||||
#else
|
||||
const __m128 mask = _mm_castsi128_ps(_mm_setr_epi32(0x00000000,0x80000000,0x00000000,0x80000000));
|
||||
return Packet2cf(_mm_add_ps(_mm_mul_ps(ei_vec4f_swizzle1(a.v, 0, 0, 2, 2), b.v),
|
||||
_mm_xor_ps(_mm_mul_ps(ei_vec4f_swizzle1(a.v, 1, 1, 3, 3),
|
||||
ei_vec4f_swizzle1(b.v, 1, 0, 3, 2)), mask)));
|
||||
return Packet2cf(_mm_add_ps(_mm_mul_ps(vec4f_swizzle1(a.v, 0, 0, 2, 2), b.v),
|
||||
_mm_xor_ps(_mm_mul_ps(vec4f_swizzle1(a.v, 1, 1, 3, 3),
|
||||
vec4f_swizzle1(b.v, 1, 0, 3, 2)), mask)));
|
||||
#endif
|
||||
}
|
||||
};
|
||||
|
||||
template<> struct ei_conj_helper<Packet2cf, Packet2cf, true,true>
|
||||
template<> struct conj_helper<Packet2cf, Packet2cf, true,true>
|
||||
{
|
||||
EIGEN_STRONG_INLINE Packet2cf pmadd(const Packet2cf& x, const Packet2cf& y, const Packet2cf& c) const
|
||||
{ return ei_padd(pmul(x,y),c); }
|
||||
{ return padd(pmul(x,y),c); }
|
||||
|
||||
EIGEN_STRONG_INLINE Packet2cf pmul(const Packet2cf& a, const Packet2cf& b) const
|
||||
{
|
||||
#ifdef EIGEN_VECTORIZE_SSE3
|
||||
return ei_pconj(ei_pmul(a, b));
|
||||
return pconj(pmul(a, b));
|
||||
#else
|
||||
const __m128 mask = _mm_castsi128_ps(_mm_setr_epi32(0x00000000,0x80000000,0x00000000,0x80000000));
|
||||
return Packet2cf(_mm_sub_ps(_mm_xor_ps(_mm_mul_ps(ei_vec4f_swizzle1(a.v, 0, 0, 2, 2), b.v), mask),
|
||||
_mm_mul_ps(ei_vec4f_swizzle1(a.v, 1, 1, 3, 3),
|
||||
ei_vec4f_swizzle1(b.v, 1, 0, 3, 2))));
|
||||
return Packet2cf(_mm_sub_ps(_mm_xor_ps(_mm_mul_ps(vec4f_swizzle1(a.v, 0, 0, 2, 2), b.v), mask),
|
||||
_mm_mul_ps(vec4f_swizzle1(a.v, 1, 1, 3, 3),
|
||||
vec4f_swizzle1(b.v, 1, 0, 3, 2))));
|
||||
#endif
|
||||
}
|
||||
};
|
||||
|
||||
template<> struct ei_conj_helper<Packet4f, Packet2cf, false,false>
|
||||
template<> struct conj_helper<Packet4f, Packet2cf, false,false>
|
||||
{
|
||||
EIGEN_STRONG_INLINE Packet2cf pmadd(const Packet4f& x, const Packet2cf& y, const Packet2cf& c) const
|
||||
{ return ei_padd(c, pmul(x,y)); }
|
||||
{ return padd(c, pmul(x,y)); }
|
||||
|
||||
EIGEN_STRONG_INLINE Packet2cf pmul(const Packet4f& x, const Packet2cf& y) const
|
||||
{ return Packet2cf(ei_pmul(x, y.v)); }
|
||||
{ return Packet2cf(Eigen::internal::pmul(x, y.v)); }
|
||||
};
|
||||
|
||||
template<> struct ei_conj_helper<Packet2cf, Packet4f, false,false>
|
||||
template<> struct conj_helper<Packet2cf, Packet4f, false,false>
|
||||
{
|
||||
EIGEN_STRONG_INLINE Packet2cf pmadd(const Packet2cf& x, const Packet4f& y, const Packet2cf& c) const
|
||||
{ return ei_padd(c, pmul(x,y)); }
|
||||
{ return padd(c, pmul(x,y)); }
|
||||
|
||||
EIGEN_STRONG_INLINE Packet2cf pmul(const Packet2cf& x, const Packet4f& y) const
|
||||
{ return Packet2cf(ei_pmul(x.v, y)); }
|
||||
{ return Packet2cf(Eigen::internal::pmul(x.v, y)); }
|
||||
};
|
||||
|
||||
template<> EIGEN_STRONG_INLINE Packet2cf ei_pdiv<Packet2cf>(const Packet2cf& a, const Packet2cf& b)
|
||||
template<> EIGEN_STRONG_INLINE Packet2cf pdiv<Packet2cf>(const Packet2cf& a, const Packet2cf& b)
|
||||
{
|
||||
// TODO optimize it for SSE3 and 4
|
||||
Packet2cf res = ei_conj_helper<Packet2cf,Packet2cf,false,true>().pmul(a,b);
|
||||
Packet2cf res = conj_helper<Packet2cf,Packet2cf,false,true>().pmul(a,b);
|
||||
__m128 s = _mm_mul_ps(b.v,b.v);
|
||||
return Packet2cf(_mm_div_ps(res.v,_mm_add_ps(s,_mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(s), 0xb1)))));
|
||||
}
|
||||
|
||||
EIGEN_STRONG_INLINE Packet2cf ei_pcplxflip/*<Packet2cf>*/(const Packet2cf& x)
|
||||
EIGEN_STRONG_INLINE Packet2cf pcplxflip/*<Packet2cf>*/(const Packet2cf& x)
|
||||
{
|
||||
return Packet2cf(ei_vec4f_swizzle1(x.v, 1, 0, 3, 2));
|
||||
return Packet2cf(vec4f_swizzle1(x.v, 1, 0, 3, 2));
|
||||
}
|
||||
|
||||
|
||||
@@ -238,7 +240,7 @@ struct Packet1cd
|
||||
__m128d v;
|
||||
};
|
||||
|
||||
template<> struct ei_packet_traits<std::complex<double> > : ei_default_packet_traits
|
||||
template<> struct packet_traits<std::complex<double> > : default_packet_traits
|
||||
{
|
||||
typedef Packet1cd type;
|
||||
enum {
|
||||
@@ -259,77 +261,77 @@ template<> struct ei_packet_traits<std::complex<double> > : ei_default_packet_t
|
||||
};
|
||||
};
|
||||
|
||||
template<> struct ei_unpacket_traits<Packet1cd> { typedef std::complex<double> type; enum {size=1}; };
|
||||
template<> struct unpacket_traits<Packet1cd> { typedef std::complex<double> type; enum {size=1}; };
|
||||
|
||||
template<> EIGEN_STRONG_INLINE Packet1cd ei_padd<Packet1cd>(const Packet1cd& a, const Packet1cd& b) { return Packet1cd(_mm_add_pd(a.v,b.v)); }
|
||||
template<> EIGEN_STRONG_INLINE Packet1cd ei_psub<Packet1cd>(const Packet1cd& a, const Packet1cd& b) { return Packet1cd(_mm_sub_pd(a.v,b.v)); }
|
||||
template<> EIGEN_STRONG_INLINE Packet1cd ei_pnegate(const Packet1cd& a) { return Packet1cd(ei_pnegate(a.v)); }
|
||||
template<> EIGEN_STRONG_INLINE Packet1cd ei_pconj(const Packet1cd& a)
|
||||
template<> EIGEN_STRONG_INLINE Packet1cd padd<Packet1cd>(const Packet1cd& a, const Packet1cd& b) { return Packet1cd(_mm_add_pd(a.v,b.v)); }
|
||||
template<> EIGEN_STRONG_INLINE Packet1cd psub<Packet1cd>(const Packet1cd& a, const Packet1cd& b) { return Packet1cd(_mm_sub_pd(a.v,b.v)); }
|
||||
template<> EIGEN_STRONG_INLINE Packet1cd pnegate(const Packet1cd& a) { return Packet1cd(pnegate(a.v)); }
|
||||
template<> EIGEN_STRONG_INLINE Packet1cd pconj(const Packet1cd& a)
|
||||
{
|
||||
const __m128d mask = _mm_castsi128_pd(_mm_set_epi32(0x80000000,0x0,0x0,0x0));
|
||||
return Packet1cd(_mm_xor_pd(a.v,mask));
|
||||
}
|
||||
|
||||
template<> EIGEN_STRONG_INLINE Packet1cd ei_pmul<Packet1cd>(const Packet1cd& a, const Packet1cd& b)
|
||||
template<> EIGEN_STRONG_INLINE Packet1cd pmul<Packet1cd>(const Packet1cd& a, const Packet1cd& b)
|
||||
{
|
||||
// TODO optimize it for SSE3 and 4
|
||||
#ifdef EIGEN_VECTORIZE_SSE3
|
||||
return Packet1cd(_mm_addsub_pd(_mm_mul_pd(ei_vec2d_swizzle1(a.v, 0, 0), b.v),
|
||||
_mm_mul_pd(ei_vec2d_swizzle1(a.v, 1, 1),
|
||||
ei_vec2d_swizzle1(b.v, 1, 0))));
|
||||
return Packet1cd(_mm_addsub_pd(_mm_mul_pd(vec2d_swizzle1(a.v, 0, 0), b.v),
|
||||
_mm_mul_pd(vec2d_swizzle1(a.v, 1, 1),
|
||||
vec2d_swizzle1(b.v, 1, 0))));
|
||||
#else
|
||||
const __m128d mask = _mm_castsi128_pd(_mm_set_epi32(0x0,0x0,0x80000000,0x0));
|
||||
return Packet1cd(_mm_add_pd(_mm_mul_pd(ei_vec2d_swizzle1(a.v, 0, 0), b.v),
|
||||
_mm_xor_pd(_mm_mul_pd(ei_vec2d_swizzle1(a.v, 1, 1),
|
||||
ei_vec2d_swizzle1(b.v, 1, 0)), mask)));
|
||||
return Packet1cd(_mm_add_pd(_mm_mul_pd(vec2d_swizzle1(a.v, 0, 0), b.v),
|
||||
_mm_xor_pd(_mm_mul_pd(vec2d_swizzle1(a.v, 1, 1),
|
||||
vec2d_swizzle1(b.v, 1, 0)), mask)));
|
||||
#endif
|
||||
}
|
||||
|
||||
template<> EIGEN_STRONG_INLINE Packet1cd ei_pand <Packet1cd>(const Packet1cd& a, const Packet1cd& b) { return Packet1cd(_mm_and_pd(a.v,b.v)); }
|
||||
template<> EIGEN_STRONG_INLINE Packet1cd ei_por <Packet1cd>(const Packet1cd& a, const Packet1cd& b) { return Packet1cd(_mm_or_pd(a.v,b.v)); }
|
||||
template<> EIGEN_STRONG_INLINE Packet1cd ei_pxor <Packet1cd>(const Packet1cd& a, const Packet1cd& b) { return Packet1cd(_mm_xor_pd(a.v,b.v)); }
|
||||
template<> EIGEN_STRONG_INLINE Packet1cd ei_pandnot<Packet1cd>(const Packet1cd& a, const Packet1cd& b) { return Packet1cd(_mm_andnot_pd(a.v,b.v)); }
|
||||
template<> EIGEN_STRONG_INLINE Packet1cd pand <Packet1cd>(const Packet1cd& a, const Packet1cd& b) { return Packet1cd(_mm_and_pd(a.v,b.v)); }
|
||||
template<> EIGEN_STRONG_INLINE Packet1cd por <Packet1cd>(const Packet1cd& a, const Packet1cd& b) { return Packet1cd(_mm_or_pd(a.v,b.v)); }
|
||||
template<> EIGEN_STRONG_INLINE Packet1cd pxor <Packet1cd>(const Packet1cd& a, const Packet1cd& b) { return Packet1cd(_mm_xor_pd(a.v,b.v)); }
|
||||
template<> EIGEN_STRONG_INLINE Packet1cd pandnot<Packet1cd>(const Packet1cd& a, const Packet1cd& b) { return Packet1cd(_mm_andnot_pd(a.v,b.v)); }
|
||||
|
||||
// FIXME force unaligned load, this is a temporary fix
|
||||
template<> EIGEN_STRONG_INLINE Packet1cd ei_pload <Packet1cd>(const std::complex<double>* from)
|
||||
{ EIGEN_DEBUG_ALIGNED_LOAD return Packet1cd(ei_pload<Packet2d>((const double*)from)); }
|
||||
template<> EIGEN_STRONG_INLINE Packet1cd ei_ploadu<Packet1cd>(const std::complex<double>* from)
|
||||
{ EIGEN_DEBUG_UNALIGNED_LOAD return Packet1cd(ei_ploadu<Packet2d>((const double*)from)); }
|
||||
template<> EIGEN_STRONG_INLINE Packet1cd ei_pset1<Packet1cd>(const std::complex<double>& from)
|
||||
{ /* here we really have to use unaligned loads :( */ return ei_ploadu<Packet1cd>(&from); }
|
||||
template<> EIGEN_STRONG_INLINE Packet1cd pload <Packet1cd>(const std::complex<double>* from)
|
||||
{ EIGEN_DEBUG_ALIGNED_LOAD return Packet1cd(pload<Packet2d>((const double*)from)); }
|
||||
template<> EIGEN_STRONG_INLINE Packet1cd ploadu<Packet1cd>(const std::complex<double>* from)
|
||||
{ EIGEN_DEBUG_UNALIGNED_LOAD return Packet1cd(ploadu<Packet2d>((const double*)from)); }
|
||||
template<> EIGEN_STRONG_INLINE Packet1cd pset1<Packet1cd>(const std::complex<double>& from)
|
||||
{ /* here we really have to use unaligned loads :( */ return ploadu<Packet1cd>(&from); }
|
||||
|
||||
// FIXME force unaligned store, this is a temporary fix
|
||||
template<> EIGEN_STRONG_INLINE void ei_pstore <std::complex<double> >(std::complex<double> * to, const Packet1cd& from) { EIGEN_DEBUG_ALIGNED_STORE ei_pstore((double*)to, from.v); }
|
||||
template<> EIGEN_STRONG_INLINE void ei_pstoreu<std::complex<double> >(std::complex<double> * to, const Packet1cd& from) { EIGEN_DEBUG_UNALIGNED_STORE ei_pstoreu((double*)to, from.v); }
|
||||
template<> EIGEN_STRONG_INLINE void pstore <std::complex<double> >(std::complex<double> * to, const Packet1cd& from) { EIGEN_DEBUG_ALIGNED_STORE pstore((double*)to, from.v); }
|
||||
template<> EIGEN_STRONG_INLINE void pstoreu<std::complex<double> >(std::complex<double> * to, const Packet1cd& from) { EIGEN_DEBUG_UNALIGNED_STORE pstoreu((double*)to, from.v); }
|
||||
|
||||
template<> EIGEN_STRONG_INLINE void ei_prefetch<std::complex<double> >(const std::complex<double> * addr) { _mm_prefetch((const char*)(addr), _MM_HINT_T0); }
|
||||
template<> EIGEN_STRONG_INLINE void prefetch<std::complex<double> >(const std::complex<double> * addr) { _mm_prefetch((const char*)(addr), _MM_HINT_T0); }
|
||||
|
||||
template<> EIGEN_STRONG_INLINE std::complex<double> ei_pfirst<Packet1cd>(const Packet1cd& a)
|
||||
template<> EIGEN_STRONG_INLINE std::complex<double> pfirst<Packet1cd>(const Packet1cd& a)
|
||||
{
|
||||
EIGEN_ALIGN16 double res[2];
|
||||
_mm_store_pd(res, a.v);
|
||||
return std::complex<double>(res[0],res[1]);
|
||||
}
|
||||
|
||||
template<> EIGEN_STRONG_INLINE Packet1cd ei_preverse(const Packet1cd& a) { return a; }
|
||||
template<> EIGEN_STRONG_INLINE Packet1cd preverse(const Packet1cd& a) { return a; }
|
||||
|
||||
template<> EIGEN_STRONG_INLINE std::complex<double> ei_predux<Packet1cd>(const Packet1cd& a)
|
||||
template<> EIGEN_STRONG_INLINE std::complex<double> predux<Packet1cd>(const Packet1cd& a)
|
||||
{
|
||||
return ei_pfirst(a);
|
||||
return pfirst(a);
|
||||
}
|
||||
|
||||
template<> EIGEN_STRONG_INLINE Packet1cd ei_preduxp<Packet1cd>(const Packet1cd* vecs)
|
||||
template<> EIGEN_STRONG_INLINE Packet1cd preduxp<Packet1cd>(const Packet1cd* vecs)
|
||||
{
|
||||
return vecs[0];
|
||||
}
|
||||
|
||||
template<> EIGEN_STRONG_INLINE std::complex<double> ei_predux_mul<Packet1cd>(const Packet1cd& a)
|
||||
template<> EIGEN_STRONG_INLINE std::complex<double> predux_mul<Packet1cd>(const Packet1cd& a)
|
||||
{
|
||||
return ei_pfirst(a);
|
||||
return pfirst(a);
|
||||
}
|
||||
|
||||
template<int Offset>
|
||||
struct ei_palign_impl<Offset,Packet1cd>
|
||||
struct palign_impl<Offset,Packet1cd>
|
||||
{
|
||||
EIGEN_STRONG_INLINE static void run(Packet1cd& /*first*/, const Packet1cd& /*second*/)
|
||||
{
|
||||
@@ -338,89 +340,91 @@ struct ei_palign_impl<Offset,Packet1cd>
|
||||
}
|
||||
};
|
||||
|
||||
template<> struct ei_conj_helper<Packet1cd, Packet1cd, false,true>
|
||||
template<> struct conj_helper<Packet1cd, Packet1cd, false,true>
|
||||
{
|
||||
EIGEN_STRONG_INLINE Packet1cd pmadd(const Packet1cd& x, const Packet1cd& y, const Packet1cd& c) const
|
||||
{ return ei_padd(pmul(x,y),c); }
|
||||
{ return padd(pmul(x,y),c); }
|
||||
|
||||
EIGEN_STRONG_INLINE Packet1cd pmul(const Packet1cd& a, const Packet1cd& b) const
|
||||
{
|
||||
#ifdef EIGEN_VECTORIZE_SSE3
|
||||
return ei_pmul(a, ei_pconj(b));
|
||||
return pmul(a, pconj(b));
|
||||
#else
|
||||
const __m128d mask = _mm_castsi128_pd(_mm_set_epi32(0x80000000,0x0,0x0,0x0));
|
||||
return Packet1cd(_mm_add_pd(_mm_xor_pd(_mm_mul_pd(ei_vec2d_swizzle1(a.v, 0, 0), b.v), mask),
|
||||
_mm_mul_pd(ei_vec2d_swizzle1(a.v, 1, 1),
|
||||
ei_vec2d_swizzle1(b.v, 1, 0))));
|
||||
return Packet1cd(_mm_add_pd(_mm_xor_pd(_mm_mul_pd(vec2d_swizzle1(a.v, 0, 0), b.v), mask),
|
||||
_mm_mul_pd(vec2d_swizzle1(a.v, 1, 1),
|
||||
vec2d_swizzle1(b.v, 1, 0))));
|
||||
#endif
|
||||
}
|
||||
};
|
||||
|
||||
template<> struct ei_conj_helper<Packet1cd, Packet1cd, true,false>
|
||||
template<> struct conj_helper<Packet1cd, Packet1cd, true,false>
|
||||
{
|
||||
EIGEN_STRONG_INLINE Packet1cd pmadd(const Packet1cd& x, const Packet1cd& y, const Packet1cd& c) const
|
||||
{ return ei_padd(pmul(x,y),c); }
|
||||
{ return padd(pmul(x,y),c); }
|
||||
|
||||
EIGEN_STRONG_INLINE Packet1cd pmul(const Packet1cd& a, const Packet1cd& b) const
|
||||
{
|
||||
#ifdef EIGEN_VECTORIZE_SSE3
|
||||
return ei_pmul(ei_pconj(a), b);
|
||||
return pmul(pconj(a), b);
|
||||
#else
|
||||
const __m128d mask = _mm_castsi128_pd(_mm_set_epi32(0x80000000,0x0,0x0,0x0));
|
||||
return Packet1cd(_mm_add_pd(_mm_mul_pd(ei_vec2d_swizzle1(a.v, 0, 0), b.v),
|
||||
_mm_xor_pd(_mm_mul_pd(ei_vec2d_swizzle1(a.v, 1, 1),
|
||||
ei_vec2d_swizzle1(b.v, 1, 0)), mask)));
|
||||
return Packet1cd(_mm_add_pd(_mm_mul_pd(vec2d_swizzle1(a.v, 0, 0), b.v),
|
||||
_mm_xor_pd(_mm_mul_pd(vec2d_swizzle1(a.v, 1, 1),
|
||||
vec2d_swizzle1(b.v, 1, 0)), mask)));
|
||||
#endif
|
||||
}
|
||||
};
|
||||
|
||||
template<> struct ei_conj_helper<Packet1cd, Packet1cd, true,true>
|
||||
template<> struct conj_helper<Packet1cd, Packet1cd, true,true>
|
||||
{
|
||||
EIGEN_STRONG_INLINE Packet1cd pmadd(const Packet1cd& x, const Packet1cd& y, const Packet1cd& c) const
|
||||
{ return ei_padd(pmul(x,y),c); }
|
||||
{ return padd(pmul(x,y),c); }
|
||||
|
||||
EIGEN_STRONG_INLINE Packet1cd pmul(const Packet1cd& a, const Packet1cd& b) const
|
||||
{
|
||||
#ifdef EIGEN_VECTORIZE_SSE3
|
||||
return ei_pconj(ei_pmul(a, b));
|
||||
return pconj(pmul(a, b));
|
||||
#else
|
||||
const __m128d mask = _mm_castsi128_pd(_mm_set_epi32(0x80000000,0x0,0x0,0x0));
|
||||
return Packet1cd(_mm_sub_pd(_mm_xor_pd(_mm_mul_pd(ei_vec2d_swizzle1(a.v, 0, 0), b.v), mask),
|
||||
_mm_mul_pd(ei_vec2d_swizzle1(a.v, 1, 1),
|
||||
ei_vec2d_swizzle1(b.v, 1, 0))));
|
||||
return Packet1cd(_mm_sub_pd(_mm_xor_pd(_mm_mul_pd(vec2d_swizzle1(a.v, 0, 0), b.v), mask),
|
||||
_mm_mul_pd(vec2d_swizzle1(a.v, 1, 1),
|
||||
vec2d_swizzle1(b.v, 1, 0))));
|
||||
#endif
|
||||
}
|
||||
};
|
||||
|
||||
template<> struct ei_conj_helper<Packet2d, Packet1cd, false,false>
|
||||
template<> struct conj_helper<Packet2d, Packet1cd, false,false>
|
||||
{
|
||||
EIGEN_STRONG_INLINE Packet1cd pmadd(const Packet2d& x, const Packet1cd& y, const Packet1cd& c) const
|
||||
{ return ei_padd(c, pmul(x,y)); }
|
||||
{ return padd(c, pmul(x,y)); }
|
||||
|
||||
EIGEN_STRONG_INLINE Packet1cd pmul(const Packet2d& x, const Packet1cd& y) const
|
||||
{ return Packet1cd(ei_pmul(x, y.v)); }
|
||||
{ return Packet1cd(Eigen::internal::pmul(x, y.v)); }
|
||||
};
|
||||
|
||||
template<> struct ei_conj_helper<Packet1cd, Packet2d, false,false>
|
||||
template<> struct conj_helper<Packet1cd, Packet2d, false,false>
|
||||
{
|
||||
EIGEN_STRONG_INLINE Packet1cd pmadd(const Packet1cd& x, const Packet2d& y, const Packet1cd& c) const
|
||||
{ return ei_padd(c, pmul(x,y)); }
|
||||
{ return padd(c, pmul(x,y)); }
|
||||
|
||||
EIGEN_STRONG_INLINE Packet1cd pmul(const Packet1cd& x, const Packet2d& y) const
|
||||
{ return Packet1cd(ei_pmul(x.v, y)); }
|
||||
{ return Packet1cd(Eigen::internal::pmul(x.v, y)); }
|
||||
};
|
||||
|
||||
template<> EIGEN_STRONG_INLINE Packet1cd ei_pdiv<Packet1cd>(const Packet1cd& a, const Packet1cd& b)
|
||||
template<> EIGEN_STRONG_INLINE Packet1cd pdiv<Packet1cd>(const Packet1cd& a, const Packet1cd& b)
|
||||
{
|
||||
// TODO optimize it for SSE3 and 4
|
||||
Packet1cd res = ei_conj_helper<Packet1cd,Packet1cd,false,true>().pmul(a,b);
|
||||
Packet1cd res = conj_helper<Packet1cd,Packet1cd,false,true>().pmul(a,b);
|
||||
__m128d s = _mm_mul_pd(b.v,b.v);
|
||||
return Packet1cd(_mm_div_pd(res.v, _mm_add_pd(s,_mm_shuffle_pd(s, s, 0x1))));
|
||||
}
|
||||
|
||||
EIGEN_STRONG_INLINE Packet1cd ei_pcplxflip/*<Packet1cd>*/(const Packet1cd& x)
|
||||
EIGEN_STRONG_INLINE Packet1cd pcplxflip/*<Packet1cd>*/(const Packet1cd& x)
|
||||
{
|
||||
return Packet1cd(ei_preverse(x.v));
|
||||
return Packet1cd(preverse(x.v));
|
||||
}
|
||||
|
||||
} // end namespace internal
|
||||
|
||||
#endif // EIGEN_COMPLEX_SSE_H
|
||||
|
||||
@@ -30,8 +30,10 @@
|
||||
#ifndef EIGEN_MATH_FUNCTIONS_SSE_H
|
||||
#define EIGEN_MATH_FUNCTIONS_SSE_H
|
||||
|
||||
namespace internal {
|
||||
|
||||
template<> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED
|
||||
Packet4f ei_plog<Packet4f>(const Packet4f& _x)
|
||||
Packet4f plog<Packet4f>(const Packet4f& _x)
|
||||
{
|
||||
Packet4f x = _x;
|
||||
_EIGEN_DECLARE_CONST_Packet4f(1 , 1.0f);
|
||||
@@ -64,15 +66,15 @@ Packet4f ei_plog<Packet4f>(const Packet4f& _x)
|
||||
|
||||
Packet4f invalid_mask = _mm_cmple_ps(x, _mm_setzero_ps());
|
||||
|
||||
x = ei_pmax(x, ei_p4f_min_norm_pos); /* cut off denormalized stuff */
|
||||
x = pmax(x, p4f_min_norm_pos); /* cut off denormalized stuff */
|
||||
emm0 = _mm_srli_epi32(_mm_castps_si128(x), 23);
|
||||
|
||||
/* keep only the fractional part */
|
||||
x = _mm_and_ps(x, ei_p4f_inv_mant_mask);
|
||||
x = _mm_or_ps(x, ei_p4f_half);
|
||||
x = _mm_and_ps(x, p4f_inv_mant_mask);
|
||||
x = _mm_or_ps(x, p4f_half);
|
||||
|
||||
emm0 = _mm_sub_epi32(emm0, ei_p4i_0x7f);
|
||||
Packet4f e = ei_padd(_mm_cvtepi32_ps(emm0), ei_p4f_1);
|
||||
emm0 = _mm_sub_epi32(emm0, p4i_0x7f);
|
||||
Packet4f e = padd(_mm_cvtepi32_ps(emm0), p4f_1);
|
||||
|
||||
/* part2:
|
||||
if( x < SQRTHF ) {
|
||||
@@ -80,38 +82,38 @@ Packet4f ei_plog<Packet4f>(const Packet4f& _x)
|
||||
x = x + x - 1.0;
|
||||
} else { x = x - 1.0; }
|
||||
*/
|
||||
Packet4f mask = _mm_cmplt_ps(x, ei_p4f_cephes_SQRTHF);
|
||||
Packet4f mask = _mm_cmplt_ps(x, p4f_cephes_SQRTHF);
|
||||
Packet4f tmp = _mm_and_ps(x, mask);
|
||||
x = ei_psub(x, ei_p4f_1);
|
||||
e = ei_psub(e, _mm_and_ps(ei_p4f_1, mask));
|
||||
x = ei_padd(x, tmp);
|
||||
x = psub(x, p4f_1);
|
||||
e = psub(e, _mm_and_ps(p4f_1, mask));
|
||||
x = padd(x, tmp);
|
||||
|
||||
Packet4f x2 = ei_pmul(x,x);
|
||||
Packet4f x3 = ei_pmul(x2,x);
|
||||
Packet4f x2 = pmul(x,x);
|
||||
Packet4f x3 = pmul(x2,x);
|
||||
|
||||
Packet4f y, y1, y2;
|
||||
y = ei_pmadd(ei_p4f_cephes_log_p0, x, ei_p4f_cephes_log_p1);
|
||||
y1 = ei_pmadd(ei_p4f_cephes_log_p3, x, ei_p4f_cephes_log_p4);
|
||||
y2 = ei_pmadd(ei_p4f_cephes_log_p6, x, ei_p4f_cephes_log_p7);
|
||||
y = ei_pmadd(y , x, ei_p4f_cephes_log_p2);
|
||||
y1 = ei_pmadd(y1, x, ei_p4f_cephes_log_p5);
|
||||
y2 = ei_pmadd(y2, x, ei_p4f_cephes_log_p8);
|
||||
y = ei_pmadd(y, x3, y1);
|
||||
y = ei_pmadd(y, x3, y2);
|
||||
y = ei_pmul(y, x3);
|
||||
y = pmadd(p4f_cephes_log_p0, x, p4f_cephes_log_p1);
|
||||
y1 = pmadd(p4f_cephes_log_p3, x, p4f_cephes_log_p4);
|
||||
y2 = pmadd(p4f_cephes_log_p6, x, p4f_cephes_log_p7);
|
||||
y = pmadd(y , x, p4f_cephes_log_p2);
|
||||
y1 = pmadd(y1, x, p4f_cephes_log_p5);
|
||||
y2 = pmadd(y2, x, p4f_cephes_log_p8);
|
||||
y = pmadd(y, x3, y1);
|
||||
y = pmadd(y, x3, y2);
|
||||
y = pmul(y, x3);
|
||||
|
||||
y1 = ei_pmul(e, ei_p4f_cephes_log_q1);
|
||||
tmp = ei_pmul(x2, ei_p4f_half);
|
||||
y = ei_padd(y, y1);
|
||||
x = ei_psub(x, tmp);
|
||||
y2 = ei_pmul(e, ei_p4f_cephes_log_q2);
|
||||
x = ei_padd(x, y);
|
||||
x = ei_padd(x, y2);
|
||||
y1 = pmul(e, p4f_cephes_log_q1);
|
||||
tmp = pmul(x2, p4f_half);
|
||||
y = padd(y, y1);
|
||||
x = psub(x, tmp);
|
||||
y2 = pmul(e, p4f_cephes_log_q2);
|
||||
x = padd(x, y);
|
||||
x = padd(x, y2);
|
||||
return _mm_or_ps(x, invalid_mask); // negative arg will be NAN
|
||||
}
|
||||
|
||||
template<> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED
|
||||
Packet4f ei_pexp<Packet4f>(const Packet4f& _x)
|
||||
Packet4f pexp<Packet4f>(const Packet4f& _x)
|
||||
{
|
||||
Packet4f x = _x;
|
||||
_EIGEN_DECLARE_CONST_Packet4f(1 , 1.0f);
|
||||
@@ -137,40 +139,40 @@ Packet4f ei_pexp<Packet4f>(const Packet4f& _x)
|
||||
Packet4i emm0;
|
||||
|
||||
// clamp x
|
||||
x = ei_pmax(ei_pmin(x, ei_p4f_exp_hi), ei_p4f_exp_lo);
|
||||
x = pmax(pmin(x, p4f_exp_hi), p4f_exp_lo);
|
||||
|
||||
/* express exp(x) as exp(g + n*log(2)) */
|
||||
fx = ei_pmadd(x, ei_p4f_cephes_LOG2EF, ei_p4f_half);
|
||||
fx = pmadd(x, p4f_cephes_LOG2EF, p4f_half);
|
||||
|
||||
/* how to perform a floorf with SSE: just below */
|
||||
emm0 = _mm_cvttps_epi32(fx);
|
||||
tmp = _mm_cvtepi32_ps(emm0);
|
||||
/* if greater, substract 1 */
|
||||
Packet4f mask = _mm_cmpgt_ps(tmp, fx);
|
||||
mask = _mm_and_ps(mask, ei_p4f_1);
|
||||
fx = ei_psub(tmp, mask);
|
||||
mask = _mm_and_ps(mask, p4f_1);
|
||||
fx = psub(tmp, mask);
|
||||
|
||||
tmp = ei_pmul(fx, ei_p4f_cephes_exp_C1);
|
||||
Packet4f z = ei_pmul(fx, ei_p4f_cephes_exp_C2);
|
||||
x = ei_psub(x, tmp);
|
||||
x = ei_psub(x, z);
|
||||
tmp = pmul(fx, p4f_cephes_exp_C1);
|
||||
Packet4f z = pmul(fx, p4f_cephes_exp_C2);
|
||||
x = psub(x, tmp);
|
||||
x = psub(x, z);
|
||||
|
||||
z = ei_pmul(x,x);
|
||||
z = pmul(x,x);
|
||||
|
||||
Packet4f y = ei_p4f_cephes_exp_p0;
|
||||
y = ei_pmadd(y, x, ei_p4f_cephes_exp_p1);
|
||||
y = ei_pmadd(y, x, ei_p4f_cephes_exp_p2);
|
||||
y = ei_pmadd(y, x, ei_p4f_cephes_exp_p3);
|
||||
y = ei_pmadd(y, x, ei_p4f_cephes_exp_p4);
|
||||
y = ei_pmadd(y, x, ei_p4f_cephes_exp_p5);
|
||||
y = ei_pmadd(y, z, x);
|
||||
y = ei_padd(y, ei_p4f_1);
|
||||
Packet4f y = p4f_cephes_exp_p0;
|
||||
y = pmadd(y, x, p4f_cephes_exp_p1);
|
||||
y = pmadd(y, x, p4f_cephes_exp_p2);
|
||||
y = pmadd(y, x, p4f_cephes_exp_p3);
|
||||
y = pmadd(y, x, p4f_cephes_exp_p4);
|
||||
y = pmadd(y, x, p4f_cephes_exp_p5);
|
||||
y = pmadd(y, z, x);
|
||||
y = padd(y, p4f_1);
|
||||
|
||||
/* build 2^n */
|
||||
emm0 = _mm_cvttps_epi32(fx);
|
||||
emm0 = _mm_add_epi32(emm0, ei_p4i_0x7f);
|
||||
emm0 = _mm_add_epi32(emm0, p4i_0x7f);
|
||||
emm0 = _mm_slli_epi32(emm0, 23);
|
||||
return ei_pmul(y, _mm_castsi128_ps(emm0));
|
||||
return pmul(y, _mm_castsi128_ps(emm0));
|
||||
}
|
||||
|
||||
/* evaluation of 4 sines at onces, using SSE2 intrinsics.
|
||||
@@ -186,7 +188,7 @@ Packet4f ei_pexp<Packet4f>(const Packet4f& _x)
|
||||
*/
|
||||
|
||||
template<> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED
|
||||
Packet4f ei_psin<Packet4f>(const Packet4f& _x)
|
||||
Packet4f psin<Packet4f>(const Packet4f& _x)
|
||||
{
|
||||
Packet4f x = _x;
|
||||
_EIGEN_DECLARE_CONST_Packet4f(1 , 1.0f);
|
||||
@@ -215,24 +217,24 @@ Packet4f ei_psin<Packet4f>(const Packet4f& _x)
|
||||
Packet4i emm0, emm2;
|
||||
sign_bit = x;
|
||||
/* take the absolute value */
|
||||
x = ei_pabs(x);
|
||||
x = pabs(x);
|
||||
|
||||
/* take the modulo */
|
||||
|
||||
/* extract the sign bit (upper one) */
|
||||
sign_bit = _mm_and_ps(sign_bit, ei_p4f_sign_mask);
|
||||
sign_bit = _mm_and_ps(sign_bit, p4f_sign_mask);
|
||||
|
||||
/* scale by 4/Pi */
|
||||
y = ei_pmul(x, ei_p4f_cephes_FOPI);
|
||||
y = pmul(x, p4f_cephes_FOPI);
|
||||
|
||||
/* store the integer part of y in mm0 */
|
||||
emm2 = _mm_cvttps_epi32(y);
|
||||
/* j=(j+1) & (~1) (see the cephes sources) */
|
||||
emm2 = _mm_add_epi32(emm2, ei_p4i_1);
|
||||
emm2 = _mm_and_si128(emm2, ei_p4i_not1);
|
||||
emm2 = _mm_add_epi32(emm2, p4i_1);
|
||||
emm2 = _mm_and_si128(emm2, p4i_not1);
|
||||
y = _mm_cvtepi32_ps(emm2);
|
||||
/* get the swap sign flag */
|
||||
emm0 = _mm_and_si128(emm2, ei_p4i_4);
|
||||
emm0 = _mm_and_si128(emm2, p4i_4);
|
||||
emm0 = _mm_slli_epi32(emm0, 29);
|
||||
/* get the polynom selection mask
|
||||
there is one polynom for 0 <= x <= Pi/4
|
||||
@@ -240,7 +242,7 @@ Packet4f ei_psin<Packet4f>(const Packet4f& _x)
|
||||
|
||||
Both branches will be computed.
|
||||
*/
|
||||
emm2 = _mm_and_si128(emm2, ei_p4i_2);
|
||||
emm2 = _mm_and_si128(emm2, p4i_2);
|
||||
emm2 = _mm_cmpeq_epi32(emm2, _mm_setzero_si128());
|
||||
|
||||
Packet4f swap_sign_bit = _mm_castsi128_ps(emm0);
|
||||
@@ -249,33 +251,33 @@ Packet4f ei_psin<Packet4f>(const Packet4f& _x)
|
||||
|
||||
/* The magic pass: "Extended precision modular arithmetic"
|
||||
x = ((x - y * DP1) - y * DP2) - y * DP3; */
|
||||
xmm1 = ei_pmul(y, ei_p4f_minus_cephes_DP1);
|
||||
xmm2 = ei_pmul(y, ei_p4f_minus_cephes_DP2);
|
||||
xmm3 = ei_pmul(y, ei_p4f_minus_cephes_DP3);
|
||||
x = ei_padd(x, xmm1);
|
||||
x = ei_padd(x, xmm2);
|
||||
x = ei_padd(x, xmm3);
|
||||
xmm1 = pmul(y, p4f_minus_cephes_DP1);
|
||||
xmm2 = pmul(y, p4f_minus_cephes_DP2);
|
||||
xmm3 = pmul(y, p4f_minus_cephes_DP3);
|
||||
x = padd(x, xmm1);
|
||||
x = padd(x, xmm2);
|
||||
x = padd(x, xmm3);
|
||||
|
||||
/* Evaluate the first polynom (0 <= x <= Pi/4) */
|
||||
y = ei_p4f_coscof_p0;
|
||||
y = p4f_coscof_p0;
|
||||
Packet4f z = _mm_mul_ps(x,x);
|
||||
|
||||
y = ei_pmadd(y, z, ei_p4f_coscof_p1);
|
||||
y = ei_pmadd(y, z, ei_p4f_coscof_p2);
|
||||
y = ei_pmul(y, z);
|
||||
y = ei_pmul(y, z);
|
||||
Packet4f tmp = ei_pmul(z, ei_p4f_half);
|
||||
y = ei_psub(y, tmp);
|
||||
y = ei_padd(y, ei_p4f_1);
|
||||
y = pmadd(y, z, p4f_coscof_p1);
|
||||
y = pmadd(y, z, p4f_coscof_p2);
|
||||
y = pmul(y, z);
|
||||
y = pmul(y, z);
|
||||
Packet4f tmp = pmul(z, p4f_half);
|
||||
y = psub(y, tmp);
|
||||
y = padd(y, p4f_1);
|
||||
|
||||
/* Evaluate the second polynom (Pi/4 <= x <= 0) */
|
||||
|
||||
Packet4f y2 = ei_p4f_sincof_p0;
|
||||
y2 = ei_pmadd(y2, z, ei_p4f_sincof_p1);
|
||||
y2 = ei_pmadd(y2, z, ei_p4f_sincof_p2);
|
||||
y2 = ei_pmul(y2, z);
|
||||
y2 = ei_pmul(y2, x);
|
||||
y2 = ei_padd(y2, x);
|
||||
Packet4f y2 = p4f_sincof_p0;
|
||||
y2 = pmadd(y2, z, p4f_sincof_p1);
|
||||
y2 = pmadd(y2, z, p4f_sincof_p2);
|
||||
y2 = pmul(y2, z);
|
||||
y2 = pmul(y2, x);
|
||||
y2 = padd(y2, x);
|
||||
|
||||
/* select the correct result from the two polynoms */
|
||||
y2 = _mm_and_ps(poly_mask, y2);
|
||||
@@ -285,9 +287,9 @@ Packet4f ei_psin<Packet4f>(const Packet4f& _x)
|
||||
return _mm_xor_ps(y, sign_bit);
|
||||
}
|
||||
|
||||
/* almost the same as ei_psin */
|
||||
/* almost the same as psin */
|
||||
template<> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED
|
||||
Packet4f ei_pcos<Packet4f>(const Packet4f& _x)
|
||||
Packet4f pcos<Packet4f>(const Packet4f& _x)
|
||||
{
|
||||
Packet4f x = _x;
|
||||
_EIGEN_DECLARE_CONST_Packet4f(1 , 1.0f);
|
||||
@@ -312,25 +314,25 @@ Packet4f ei_pcos<Packet4f>(const Packet4f& _x)
|
||||
Packet4f xmm1, xmm2 = _mm_setzero_ps(), xmm3, y;
|
||||
Packet4i emm0, emm2;
|
||||
|
||||
x = ei_pabs(x);
|
||||
x = pabs(x);
|
||||
|
||||
/* scale by 4/Pi */
|
||||
y = ei_pmul(x, ei_p4f_cephes_FOPI);
|
||||
y = pmul(x, p4f_cephes_FOPI);
|
||||
|
||||
/* get the integer part of y */
|
||||
emm2 = _mm_cvttps_epi32(y);
|
||||
/* j=(j+1) & (~1) (see the cephes sources) */
|
||||
emm2 = _mm_add_epi32(emm2, ei_p4i_1);
|
||||
emm2 = _mm_and_si128(emm2, ei_p4i_not1);
|
||||
emm2 = _mm_add_epi32(emm2, p4i_1);
|
||||
emm2 = _mm_and_si128(emm2, p4i_not1);
|
||||
y = _mm_cvtepi32_ps(emm2);
|
||||
|
||||
emm2 = _mm_sub_epi32(emm2, ei_p4i_2);
|
||||
emm2 = _mm_sub_epi32(emm2, p4i_2);
|
||||
|
||||
/* get the swap sign flag */
|
||||
emm0 = _mm_andnot_si128(emm2, ei_p4i_4);
|
||||
emm0 = _mm_andnot_si128(emm2, p4i_4);
|
||||
emm0 = _mm_slli_epi32(emm0, 29);
|
||||
/* get the polynom selection mask */
|
||||
emm2 = _mm_and_si128(emm2, ei_p4i_2);
|
||||
emm2 = _mm_and_si128(emm2, p4i_2);
|
||||
emm2 = _mm_cmpeq_epi32(emm2, _mm_setzero_si128());
|
||||
|
||||
Packet4f sign_bit = _mm_castsi128_ps(emm0);
|
||||
@@ -338,31 +340,31 @@ Packet4f ei_pcos<Packet4f>(const Packet4f& _x)
|
||||
|
||||
/* The magic pass: "Extended precision modular arithmetic"
|
||||
x = ((x - y * DP1) - y * DP2) - y * DP3; */
|
||||
xmm1 = ei_pmul(y, ei_p4f_minus_cephes_DP1);
|
||||
xmm2 = ei_pmul(y, ei_p4f_minus_cephes_DP2);
|
||||
xmm3 = ei_pmul(y, ei_p4f_minus_cephes_DP3);
|
||||
x = ei_padd(x, xmm1);
|
||||
x = ei_padd(x, xmm2);
|
||||
x = ei_padd(x, xmm3);
|
||||
xmm1 = pmul(y, p4f_minus_cephes_DP1);
|
||||
xmm2 = pmul(y, p4f_minus_cephes_DP2);
|
||||
xmm3 = pmul(y, p4f_minus_cephes_DP3);
|
||||
x = padd(x, xmm1);
|
||||
x = padd(x, xmm2);
|
||||
x = padd(x, xmm3);
|
||||
|
||||
/* Evaluate the first polynom (0 <= x <= Pi/4) */
|
||||
y = ei_p4f_coscof_p0;
|
||||
Packet4f z = ei_pmul(x,x);
|
||||
y = p4f_coscof_p0;
|
||||
Packet4f z = pmul(x,x);
|
||||
|
||||
y = ei_pmadd(y,z,ei_p4f_coscof_p1);
|
||||
y = ei_pmadd(y,z,ei_p4f_coscof_p2);
|
||||
y = ei_pmul(y, z);
|
||||
y = ei_pmul(y, z);
|
||||
Packet4f tmp = _mm_mul_ps(z, ei_p4f_half);
|
||||
y = ei_psub(y, tmp);
|
||||
y = ei_padd(y, ei_p4f_1);
|
||||
y = pmadd(y,z,p4f_coscof_p1);
|
||||
y = pmadd(y,z,p4f_coscof_p2);
|
||||
y = pmul(y, z);
|
||||
y = pmul(y, z);
|
||||
Packet4f tmp = _mm_mul_ps(z, p4f_half);
|
||||
y = psub(y, tmp);
|
||||
y = padd(y, p4f_1);
|
||||
|
||||
/* Evaluate the second polynom (Pi/4 <= x <= 0) */
|
||||
Packet4f y2 = ei_p4f_sincof_p0;
|
||||
y2 = ei_pmadd(y2, z, ei_p4f_sincof_p1);
|
||||
y2 = ei_pmadd(y2, z, ei_p4f_sincof_p2);
|
||||
y2 = ei_pmul(y2, z);
|
||||
y2 = ei_pmadd(y2, x, x);
|
||||
Packet4f y2 = p4f_sincof_p0;
|
||||
y2 = pmadd(y2, z, p4f_sincof_p1);
|
||||
y2 = pmadd(y2, z, p4f_sincof_p2);
|
||||
y2 = pmul(y2, z);
|
||||
y2 = pmadd(y2, x, x);
|
||||
|
||||
/* select the correct result from the two polynoms */
|
||||
y2 = _mm_and_ps(poly_mask, y2);
|
||||
@@ -376,16 +378,18 @@ Packet4f ei_pcos<Packet4f>(const Packet4f& _x)
|
||||
// This is based on Quake3's fast inverse square root.
|
||||
// For detail see here: http://www.beyond3d.com/content/articles/8/
|
||||
template<> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED
|
||||
Packet4f ei_psqrt<Packet4f>(const Packet4f& _x)
|
||||
Packet4f psqrt<Packet4f>(const Packet4f& _x)
|
||||
{
|
||||
Packet4f half = ei_pmul(_x, ei_pset1<Packet4f>(.5f));
|
||||
Packet4f half = pmul(_x, pset1<Packet4f>(.5f));
|
||||
|
||||
/* select only the inverse sqrt of non-zero inputs */
|
||||
Packet4f non_zero_mask = _mm_cmpgt_ps(_x, ei_pset1<Packet4f>(std::numeric_limits<float>::epsilon()));
|
||||
Packet4f non_zero_mask = _mm_cmpgt_ps(_x, pset1<Packet4f>(std::numeric_limits<float>::epsilon()));
|
||||
Packet4f x = _mm_and_ps(non_zero_mask, _mm_rsqrt_ps(_x));
|
||||
|
||||
x = ei_pmul(x, ei_psub(ei_pset1<Packet4f>(1.5f), ei_pmul(half, ei_pmul(x,x))));
|
||||
return ei_pmul(_x,x);
|
||||
x = pmul(x, psub(pset1<Packet4f>(1.5f), pmul(half, pmul(x,x))));
|
||||
return pmul(_x,x);
|
||||
}
|
||||
|
||||
} // end namespace internal
|
||||
|
||||
#endif // EIGEN_MATH_FUNCTIONS_SSE_H
|
||||
|
||||
@@ -25,6 +25,8 @@
|
||||
#ifndef EIGEN_PACKET_MATH_SSE_H
|
||||
#define EIGEN_PACKET_MATH_SSE_H
|
||||
|
||||
namespace internal {
|
||||
|
||||
#ifndef EIGEN_CACHEFRIENDLY_PRODUCT_THRESHOLD
|
||||
#define EIGEN_CACHEFRIENDLY_PRODUCT_THRESHOLD 8
|
||||
#endif
|
||||
@@ -37,36 +39,36 @@ typedef __m128 Packet4f;
|
||||
typedef __m128i Packet4i;
|
||||
typedef __m128d Packet2d;
|
||||
|
||||
template<> struct ei_is_arithmetic<__m128> { enum { ret = true }; };
|
||||
template<> struct ei_is_arithmetic<__m128i> { enum { ret = true }; };
|
||||
template<> struct ei_is_arithmetic<__m128d> { enum { ret = true }; };
|
||||
template<> struct is_arithmetic<__m128> { enum { ret = true }; };
|
||||
template<> struct is_arithmetic<__m128i> { enum { ret = true }; };
|
||||
template<> struct is_arithmetic<__m128d> { enum { ret = true }; };
|
||||
|
||||
#define ei_vec4f_swizzle1(v,p,q,r,s) \
|
||||
#define vec4f_swizzle1(v,p,q,r,s) \
|
||||
(_mm_castsi128_ps(_mm_shuffle_epi32( _mm_castps_si128(v), ((s)<<6|(r)<<4|(q)<<2|(p)))))
|
||||
|
||||
#define ei_vec4i_swizzle1(v,p,q,r,s) \
|
||||
#define vec4i_swizzle1(v,p,q,r,s) \
|
||||
(_mm_shuffle_epi32( v, ((s)<<6|(r)<<4|(q)<<2|(p))))
|
||||
|
||||
#define ei_vec2d_swizzle1(v,p,q) \
|
||||
#define vec2d_swizzle1(v,p,q) \
|
||||
(_mm_castsi128_pd(_mm_shuffle_epi32( _mm_castpd_si128(v), ((q*2+1)<<6|(q*2)<<4|(p*2+1)<<2|(p*2)))))
|
||||
|
||||
#define ei_vec4f_swizzle2(a,b,p,q,r,s) \
|
||||
#define vec4f_swizzle2(a,b,p,q,r,s) \
|
||||
(_mm_shuffle_ps( (a), (b), ((s)<<6|(r)<<4|(q)<<2|(p))))
|
||||
|
||||
#define ei_vec4i_swizzle2(a,b,p,q,r,s) \
|
||||
#define vec4i_swizzle2(a,b,p,q,r,s) \
|
||||
(_mm_castps_si128( (_mm_shuffle_ps( _mm_castsi128_ps(a), _mm_castsi128_ps(b), ((s)<<6|(r)<<4|(q)<<2|(p))))))
|
||||
|
||||
#define _EIGEN_DECLARE_CONST_Packet4f(NAME,X) \
|
||||
const Packet4f ei_p4f_##NAME = ei_pset1<Packet4f>(X)
|
||||
const Packet4f p4f_##NAME = pset1<Packet4f>(X)
|
||||
|
||||
#define _EIGEN_DECLARE_CONST_Packet4f_FROM_INT(NAME,X) \
|
||||
const Packet4f ei_p4f_##NAME = _mm_castsi128_ps(ei_pset1<Packet4i>(X))
|
||||
const Packet4f p4f_##NAME = _mm_castsi128_ps(pset1<Packet4i>(X))
|
||||
|
||||
#define _EIGEN_DECLARE_CONST_Packet4i(NAME,X) \
|
||||
const Packet4i ei_p4i_##NAME = ei_pset1<Packet4i>(X)
|
||||
const Packet4i p4i_##NAME = pset1<Packet4i>(X)
|
||||
|
||||
|
||||
template<> struct ei_packet_traits<float> : ei_default_packet_traits
|
||||
template<> struct packet_traits<float> : default_packet_traits
|
||||
{
|
||||
typedef Packet4f type;
|
||||
enum {
|
||||
@@ -82,7 +84,7 @@ template<> struct ei_packet_traits<float> : ei_default_packet_traits
|
||||
HasSqrt = 1
|
||||
};
|
||||
};
|
||||
template<> struct ei_packet_traits<double> : ei_default_packet_traits
|
||||
template<> struct packet_traits<double> : default_packet_traits
|
||||
{
|
||||
typedef Packet2d type;
|
||||
enum {
|
||||
@@ -93,7 +95,7 @@ template<> struct ei_packet_traits<double> : ei_default_packet_traits
|
||||
HasDiv = 1
|
||||
};
|
||||
};
|
||||
template<> struct ei_packet_traits<int> : ei_default_packet_traits
|
||||
template<> struct packet_traits<int> : default_packet_traits
|
||||
{
|
||||
typedef Packet4i type;
|
||||
enum {
|
||||
@@ -104,125 +106,125 @@ template<> struct ei_packet_traits<int> : ei_default_packet_traits
|
||||
};
|
||||
};
|
||||
|
||||
template<> struct ei_unpacket_traits<Packet4f> { typedef float type; enum {size=4}; };
|
||||
template<> struct ei_unpacket_traits<Packet2d> { typedef double type; enum {size=2}; };
|
||||
template<> struct ei_unpacket_traits<Packet4i> { typedef int type; enum {size=4}; };
|
||||
template<> struct unpacket_traits<Packet4f> { typedef float type; enum {size=4}; };
|
||||
template<> struct unpacket_traits<Packet2d> { typedef double type; enum {size=2}; };
|
||||
template<> struct unpacket_traits<Packet4i> { typedef int type; enum {size=4}; };
|
||||
|
||||
#ifdef __GNUC__
|
||||
// Sometimes GCC implements _mm_set1_p* using multiple moves,
|
||||
// that is inefficient :( (e.g., see ei_gemm_pack_rhs)
|
||||
template<> EIGEN_STRONG_INLINE Packet4f ei_pset1<Packet4f>(const float& from) {
|
||||
// that is inefficient :( (e.g., see gemm_pack_rhs)
|
||||
template<> EIGEN_STRONG_INLINE Packet4f pset1<Packet4f>(const float& from) {
|
||||
Packet4f res = _mm_set_ss(from);
|
||||
return ei_vec4f_swizzle1(res,0,0,0,0);
|
||||
return vec4f_swizzle1(res,0,0,0,0);
|
||||
}
|
||||
template<> EIGEN_STRONG_INLINE Packet2d ei_pset1<Packet2d>(const double& from) {
|
||||
template<> EIGEN_STRONG_INLINE Packet2d pset1<Packet2d>(const double& from) {
|
||||
// NOTE the SSE3 intrinsic _mm_loaddup_pd is never faster but sometimes much slower
|
||||
Packet2d res = _mm_set_sd(from);
|
||||
return ei_vec2d_swizzle1(res, 0, 0);
|
||||
return vec2d_swizzle1(res, 0, 0);
|
||||
}
|
||||
#else
|
||||
template<> EIGEN_STRONG_INLINE Packet4f ei_pset1<Packet4f>(const float& from) { return _mm_set1_ps(from); }
|
||||
template<> EIGEN_STRONG_INLINE Packet2d ei_pset1<Packet2d>(const double& from) { return _mm_set1_pd(from); }
|
||||
template<> EIGEN_STRONG_INLINE Packet4f pset1<Packet4f>(const float& from) { return _mm_set1_ps(from); }
|
||||
template<> EIGEN_STRONG_INLINE Packet2d pset1<Packet2d>(const double& from) { return _mm_set1_pd(from); }
|
||||
#endif
|
||||
template<> EIGEN_STRONG_INLINE Packet4i ei_pset1<Packet4i>(const int& from) { return _mm_set1_epi32(from); }
|
||||
template<> EIGEN_STRONG_INLINE Packet4i pset1<Packet4i>(const int& from) { return _mm_set1_epi32(from); }
|
||||
|
||||
template<> EIGEN_STRONG_INLINE Packet4f ei_plset<float>(const float& a) { return _mm_add_ps(ei_pset1<Packet4f>(a), _mm_set_ps(3,2,1,0)); }
|
||||
template<> EIGEN_STRONG_INLINE Packet2d ei_plset<double>(const double& a) { return _mm_add_pd(ei_pset1<Packet2d>(a),_mm_set_pd(1,0)); }
|
||||
template<> EIGEN_STRONG_INLINE Packet4i ei_plset<int>(const int& a) { return _mm_add_epi32(ei_pset1<Packet4i>(a),_mm_set_epi32(3,2,1,0)); }
|
||||
template<> EIGEN_STRONG_INLINE Packet4f plset<float>(const float& a) { return _mm_add_ps(pset1<Packet4f>(a), _mm_set_ps(3,2,1,0)); }
|
||||
template<> EIGEN_STRONG_INLINE Packet2d plset<double>(const double& a) { return _mm_add_pd(pset1<Packet2d>(a),_mm_set_pd(1,0)); }
|
||||
template<> EIGEN_STRONG_INLINE Packet4i plset<int>(const int& a) { return _mm_add_epi32(pset1<Packet4i>(a),_mm_set_epi32(3,2,1,0)); }
|
||||
|
||||
template<> EIGEN_STRONG_INLINE Packet4f ei_padd<Packet4f>(const Packet4f& a, const Packet4f& b) { return _mm_add_ps(a,b); }
|
||||
template<> EIGEN_STRONG_INLINE Packet2d ei_padd<Packet2d>(const Packet2d& a, const Packet2d& b) { return _mm_add_pd(a,b); }
|
||||
template<> EIGEN_STRONG_INLINE Packet4i ei_padd<Packet4i>(const Packet4i& a, const Packet4i& b) { return _mm_add_epi32(a,b); }
|
||||
template<> EIGEN_STRONG_INLINE Packet4f padd<Packet4f>(const Packet4f& a, const Packet4f& b) { return _mm_add_ps(a,b); }
|
||||
template<> EIGEN_STRONG_INLINE Packet2d padd<Packet2d>(const Packet2d& a, const Packet2d& b) { return _mm_add_pd(a,b); }
|
||||
template<> EIGEN_STRONG_INLINE Packet4i padd<Packet4i>(const Packet4i& a, const Packet4i& b) { return _mm_add_epi32(a,b); }
|
||||
|
||||
template<> EIGEN_STRONG_INLINE Packet4f ei_psub<Packet4f>(const Packet4f& a, const Packet4f& b) { return _mm_sub_ps(a,b); }
|
||||
template<> EIGEN_STRONG_INLINE Packet2d ei_psub<Packet2d>(const Packet2d& a, const Packet2d& b) { return _mm_sub_pd(a,b); }
|
||||
template<> EIGEN_STRONG_INLINE Packet4i ei_psub<Packet4i>(const Packet4i& a, const Packet4i& b) { return _mm_sub_epi32(a,b); }
|
||||
template<> EIGEN_STRONG_INLINE Packet4f psub<Packet4f>(const Packet4f& a, const Packet4f& b) { return _mm_sub_ps(a,b); }
|
||||
template<> EIGEN_STRONG_INLINE Packet2d psub<Packet2d>(const Packet2d& a, const Packet2d& b) { return _mm_sub_pd(a,b); }
|
||||
template<> EIGEN_STRONG_INLINE Packet4i psub<Packet4i>(const Packet4i& a, const Packet4i& b) { return _mm_sub_epi32(a,b); }
|
||||
|
||||
template<> EIGEN_STRONG_INLINE Packet4f ei_pnegate(const Packet4f& a)
|
||||
template<> EIGEN_STRONG_INLINE Packet4f pnegate(const Packet4f& a)
|
||||
{
|
||||
const Packet4f mask = _mm_castsi128_ps(_mm_setr_epi32(0x80000000,0x80000000,0x80000000,0x80000000));
|
||||
return _mm_xor_ps(a,mask);
|
||||
}
|
||||
template<> EIGEN_STRONG_INLINE Packet2d ei_pnegate(const Packet2d& a)
|
||||
template<> EIGEN_STRONG_INLINE Packet2d pnegate(const Packet2d& a)
|
||||
{
|
||||
const Packet2d mask = _mm_castsi128_pd(_mm_setr_epi32(0x0,0x80000000,0x0,0x80000000));
|
||||
return _mm_xor_pd(a,mask);
|
||||
}
|
||||
template<> EIGEN_STRONG_INLINE Packet4i ei_pnegate(const Packet4i& a)
|
||||
template<> EIGEN_STRONG_INLINE Packet4i pnegate(const Packet4i& a)
|
||||
{
|
||||
return ei_psub(_mm_setr_epi32(0,0,0,0), a);
|
||||
return psub(_mm_setr_epi32(0,0,0,0), a);
|
||||
}
|
||||
|
||||
template<> EIGEN_STRONG_INLINE Packet4f ei_pmul<Packet4f>(const Packet4f& a, const Packet4f& b) { return _mm_mul_ps(a,b); }
|
||||
template<> EIGEN_STRONG_INLINE Packet2d ei_pmul<Packet2d>(const Packet2d& a, const Packet2d& b) { return _mm_mul_pd(a,b); }
|
||||
template<> EIGEN_STRONG_INLINE Packet4i ei_pmul<Packet4i>(const Packet4i& a, const Packet4i& b)
|
||||
template<> EIGEN_STRONG_INLINE Packet4f pmul<Packet4f>(const Packet4f& a, const Packet4f& b) { return _mm_mul_ps(a,b); }
|
||||
template<> EIGEN_STRONG_INLINE Packet2d pmul<Packet2d>(const Packet2d& a, const Packet2d& b) { return _mm_mul_pd(a,b); }
|
||||
template<> EIGEN_STRONG_INLINE Packet4i pmul<Packet4i>(const Packet4i& a, const Packet4i& b)
|
||||
{
|
||||
#ifdef EIGEN_VECTORIZE_SSE4_1
|
||||
return _mm_mullo_epi32(a,b);
|
||||
#else
|
||||
// this version is slightly faster than 4 scalar products
|
||||
return ei_vec4i_swizzle1(
|
||||
ei_vec4i_swizzle2(
|
||||
return vec4i_swizzle1(
|
||||
vec4i_swizzle2(
|
||||
_mm_mul_epu32(a,b),
|
||||
_mm_mul_epu32(ei_vec4i_swizzle1(a,1,0,3,2),
|
||||
ei_vec4i_swizzle1(b,1,0,3,2)),
|
||||
_mm_mul_epu32(vec4i_swizzle1(a,1,0,3,2),
|
||||
vec4i_swizzle1(b,1,0,3,2)),
|
||||
0,2,0,2),
|
||||
0,2,1,3);
|
||||
#endif
|
||||
}
|
||||
|
||||
template<> EIGEN_STRONG_INLINE Packet4f ei_pdiv<Packet4f>(const Packet4f& a, const Packet4f& b) { return _mm_div_ps(a,b); }
|
||||
template<> EIGEN_STRONG_INLINE Packet2d ei_pdiv<Packet2d>(const Packet2d& a, const Packet2d& b) { return _mm_div_pd(a,b); }
|
||||
template<> EIGEN_STRONG_INLINE Packet4i ei_pdiv<Packet4i>(const Packet4i& /*a*/, const Packet4i& /*b*/)
|
||||
{ ei_assert(false && "packet integer division are not supported by SSE");
|
||||
return ei_pset1<Packet4i>(0);
|
||||
template<> EIGEN_STRONG_INLINE Packet4f pdiv<Packet4f>(const Packet4f& a, const Packet4f& b) { return _mm_div_ps(a,b); }
|
||||
template<> EIGEN_STRONG_INLINE Packet2d pdiv<Packet2d>(const Packet2d& a, const Packet2d& b) { return _mm_div_pd(a,b); }
|
||||
template<> EIGEN_STRONG_INLINE Packet4i pdiv<Packet4i>(const Packet4i& /*a*/, const Packet4i& /*b*/)
|
||||
{ eigen_assert(false && "packet integer division are not supported by SSE");
|
||||
return pset1<Packet4i>(0);
|
||||
}
|
||||
|
||||
// for some weird raisons, it has to be overloaded for packet of integers
|
||||
template<> EIGEN_STRONG_INLINE Packet4i ei_pmadd(const Packet4i& a, const Packet4i& b, const Packet4i& c) { return ei_padd(ei_pmul(a,b), c); }
|
||||
template<> EIGEN_STRONG_INLINE Packet4i pmadd(const Packet4i& a, const Packet4i& b, const Packet4i& c) { return padd(pmul(a,b), c); }
|
||||
|
||||
template<> EIGEN_STRONG_INLINE Packet4f ei_pmin<Packet4f>(const Packet4f& a, const Packet4f& b) { return _mm_min_ps(a,b); }
|
||||
template<> EIGEN_STRONG_INLINE Packet2d ei_pmin<Packet2d>(const Packet2d& a, const Packet2d& b) { return _mm_min_pd(a,b); }
|
||||
template<> EIGEN_STRONG_INLINE Packet4i ei_pmin<Packet4i>(const Packet4i& a, const Packet4i& b)
|
||||
template<> EIGEN_STRONG_INLINE Packet4f pmin<Packet4f>(const Packet4f& a, const Packet4f& b) { return _mm_min_ps(a,b); }
|
||||
template<> EIGEN_STRONG_INLINE Packet2d pmin<Packet2d>(const Packet2d& a, const Packet2d& b) { return _mm_min_pd(a,b); }
|
||||
template<> EIGEN_STRONG_INLINE Packet4i pmin<Packet4i>(const Packet4i& a, const Packet4i& b)
|
||||
{
|
||||
// after some bench, this version *is* faster than a scalar implementation
|
||||
Packet4i mask = _mm_cmplt_epi32(a,b);
|
||||
return _mm_or_si128(_mm_and_si128(mask,a),_mm_andnot_si128(mask,b));
|
||||
}
|
||||
|
||||
template<> EIGEN_STRONG_INLINE Packet4f ei_pmax<Packet4f>(const Packet4f& a, const Packet4f& b) { return _mm_max_ps(a,b); }
|
||||
template<> EIGEN_STRONG_INLINE Packet2d ei_pmax<Packet2d>(const Packet2d& a, const Packet2d& b) { return _mm_max_pd(a,b); }
|
||||
template<> EIGEN_STRONG_INLINE Packet4i ei_pmax<Packet4i>(const Packet4i& a, const Packet4i& b)
|
||||
template<> EIGEN_STRONG_INLINE Packet4f pmax<Packet4f>(const Packet4f& a, const Packet4f& b) { return _mm_max_ps(a,b); }
|
||||
template<> EIGEN_STRONG_INLINE Packet2d pmax<Packet2d>(const Packet2d& a, const Packet2d& b) { return _mm_max_pd(a,b); }
|
||||
template<> EIGEN_STRONG_INLINE Packet4i pmax<Packet4i>(const Packet4i& a, const Packet4i& b)
|
||||
{
|
||||
// after some bench, this version *is* faster than a scalar implementation
|
||||
Packet4i mask = _mm_cmpgt_epi32(a,b);
|
||||
return _mm_or_si128(_mm_and_si128(mask,a),_mm_andnot_si128(mask,b));
|
||||
}
|
||||
|
||||
template<> EIGEN_STRONG_INLINE Packet4f ei_pand<Packet4f>(const Packet4f& a, const Packet4f& b) { return _mm_and_ps(a,b); }
|
||||
template<> EIGEN_STRONG_INLINE Packet2d ei_pand<Packet2d>(const Packet2d& a, const Packet2d& b) { return _mm_and_pd(a,b); }
|
||||
template<> EIGEN_STRONG_INLINE Packet4i ei_pand<Packet4i>(const Packet4i& a, const Packet4i& b) { return _mm_and_si128(a,b); }
|
||||
template<> EIGEN_STRONG_INLINE Packet4f pand<Packet4f>(const Packet4f& a, const Packet4f& b) { return _mm_and_ps(a,b); }
|
||||
template<> EIGEN_STRONG_INLINE Packet2d pand<Packet2d>(const Packet2d& a, const Packet2d& b) { return _mm_and_pd(a,b); }
|
||||
template<> EIGEN_STRONG_INLINE Packet4i pand<Packet4i>(const Packet4i& a, const Packet4i& b) { return _mm_and_si128(a,b); }
|
||||
|
||||
template<> EIGEN_STRONG_INLINE Packet4f ei_por<Packet4f>(const Packet4f& a, const Packet4f& b) { return _mm_or_ps(a,b); }
|
||||
template<> EIGEN_STRONG_INLINE Packet2d ei_por<Packet2d>(const Packet2d& a, const Packet2d& b) { return _mm_or_pd(a,b); }
|
||||
template<> EIGEN_STRONG_INLINE Packet4i ei_por<Packet4i>(const Packet4i& a, const Packet4i& b) { return _mm_or_si128(a,b); }
|
||||
template<> EIGEN_STRONG_INLINE Packet4f por<Packet4f>(const Packet4f& a, const Packet4f& b) { return _mm_or_ps(a,b); }
|
||||
template<> EIGEN_STRONG_INLINE Packet2d por<Packet2d>(const Packet2d& a, const Packet2d& b) { return _mm_or_pd(a,b); }
|
||||
template<> EIGEN_STRONG_INLINE Packet4i por<Packet4i>(const Packet4i& a, const Packet4i& b) { return _mm_or_si128(a,b); }
|
||||
|
||||
template<> EIGEN_STRONG_INLINE Packet4f ei_pxor<Packet4f>(const Packet4f& a, const Packet4f& b) { return _mm_xor_ps(a,b); }
|
||||
template<> EIGEN_STRONG_INLINE Packet2d ei_pxor<Packet2d>(const Packet2d& a, const Packet2d& b) { return _mm_xor_pd(a,b); }
|
||||
template<> EIGEN_STRONG_INLINE Packet4i ei_pxor<Packet4i>(const Packet4i& a, const Packet4i& b) { return _mm_xor_si128(a,b); }
|
||||
template<> EIGEN_STRONG_INLINE Packet4f pxor<Packet4f>(const Packet4f& a, const Packet4f& b) { return _mm_xor_ps(a,b); }
|
||||
template<> EIGEN_STRONG_INLINE Packet2d pxor<Packet2d>(const Packet2d& a, const Packet2d& b) { return _mm_xor_pd(a,b); }
|
||||
template<> EIGEN_STRONG_INLINE Packet4i pxor<Packet4i>(const Packet4i& a, const Packet4i& b) { return _mm_xor_si128(a,b); }
|
||||
|
||||
template<> EIGEN_STRONG_INLINE Packet4f ei_pandnot<Packet4f>(const Packet4f& a, const Packet4f& b) { return _mm_andnot_ps(a,b); }
|
||||
template<> EIGEN_STRONG_INLINE Packet2d ei_pandnot<Packet2d>(const Packet2d& a, const Packet2d& b) { return _mm_andnot_pd(a,b); }
|
||||
template<> EIGEN_STRONG_INLINE Packet4i ei_pandnot<Packet4i>(const Packet4i& a, const Packet4i& b) { return _mm_andnot_si128(a,b); }
|
||||
template<> EIGEN_STRONG_INLINE Packet4f pandnot<Packet4f>(const Packet4f& a, const Packet4f& b) { return _mm_andnot_ps(a,b); }
|
||||
template<> EIGEN_STRONG_INLINE Packet2d pandnot<Packet2d>(const Packet2d& a, const Packet2d& b) { return _mm_andnot_pd(a,b); }
|
||||
template<> EIGEN_STRONG_INLINE Packet4i pandnot<Packet4i>(const Packet4i& a, const Packet4i& b) { return _mm_andnot_si128(a,b); }
|
||||
|
||||
template<> EIGEN_STRONG_INLINE Packet4f ei_pload<Packet4f>(const float* from) { EIGEN_DEBUG_ALIGNED_LOAD return _mm_load_ps(from); }
|
||||
template<> EIGEN_STRONG_INLINE Packet2d ei_pload<Packet2d>(const double* from) { EIGEN_DEBUG_ALIGNED_LOAD return _mm_load_pd(from); }
|
||||
template<> EIGEN_STRONG_INLINE Packet4i ei_pload<Packet4i>(const int* from) { EIGEN_DEBUG_ALIGNED_LOAD return _mm_load_si128(reinterpret_cast<const Packet4i*>(from)); }
|
||||
template<> EIGEN_STRONG_INLINE Packet4f pload<Packet4f>(const float* from) { EIGEN_DEBUG_ALIGNED_LOAD return _mm_load_ps(from); }
|
||||
template<> EIGEN_STRONG_INLINE Packet2d pload<Packet2d>(const double* from) { EIGEN_DEBUG_ALIGNED_LOAD return _mm_load_pd(from); }
|
||||
template<> EIGEN_STRONG_INLINE Packet4i pload<Packet4i>(const int* from) { EIGEN_DEBUG_ALIGNED_LOAD return _mm_load_si128(reinterpret_cast<const Packet4i*>(from)); }
|
||||
|
||||
#if defined(_MSC_VER)
|
||||
template<> EIGEN_STRONG_INLINE Packet4f ei_ploadu<Packet4f>(const float* from) { EIGEN_DEBUG_UNALIGNED_LOAD return _mm_loadu_ps(from); }
|
||||
template<> EIGEN_STRONG_INLINE Packet2d ei_ploadu<Packet2d>(const double* from) { EIGEN_DEBUG_UNALIGNED_LOAD return _mm_loadu_pd(from); }
|
||||
template<> EIGEN_STRONG_INLINE Packet4i ei_ploadu<Packet4i>(const int* from) { EIGEN_DEBUG_UNALIGNED_LOAD return _mm_loadu_si128(reinterpret_cast<const Packet4i*>(from)); }
|
||||
template<> EIGEN_STRONG_INLINE Packet4f ploadu<Packet4f>(const float* from) { EIGEN_DEBUG_UNALIGNED_LOAD return _mm_loadu_ps(from); }
|
||||
template<> EIGEN_STRONG_INLINE Packet2d ploadu<Packet2d>(const double* from) { EIGEN_DEBUG_UNALIGNED_LOAD return _mm_loadu_pd(from); }
|
||||
template<> EIGEN_STRONG_INLINE Packet4i ploadu<Packet4i>(const int* from) { EIGEN_DEBUG_UNALIGNED_LOAD return _mm_loadu_si128(reinterpret_cast<const Packet4i*>(from)); }
|
||||
#else
|
||||
// Fast unaligned loads. Note that here we cannot directly use intrinsics: this would
|
||||
// require pointer casting to incompatible pointer types and leads to invalid code
|
||||
@@ -230,7 +232,7 @@ template<> EIGEN_STRONG_INLINE Packet4i ei_pload<Packet4i>(const int* from)
|
||||
// a correct instruction dependency.
|
||||
// TODO: do the same for MSVC (ICC is compatible)
|
||||
// NOTE: with the code below, MSVC's compiler crashes!
|
||||
template<> EIGEN_STRONG_INLINE Packet4f ei_ploadu<Packet4f>(const float* from)
|
||||
template<> EIGEN_STRONG_INLINE Packet4f ploadu<Packet4f>(const float* from)
|
||||
{
|
||||
EIGEN_DEBUG_UNALIGNED_LOAD
|
||||
__m128d res;
|
||||
@@ -238,7 +240,7 @@ template<> EIGEN_STRONG_INLINE Packet4f ei_ploadu<Packet4f>(const float* from)
|
||||
res = _mm_loadh_pd(res, (const double*)(from+2)) ;
|
||||
return _mm_castpd_ps(res);
|
||||
}
|
||||
template<> EIGEN_STRONG_INLINE Packet2d ei_ploadu<Packet2d>(const double* from)
|
||||
template<> EIGEN_STRONG_INLINE Packet2d ploadu<Packet2d>(const double* from)
|
||||
{
|
||||
EIGEN_DEBUG_UNALIGNED_LOAD
|
||||
__m128d res;
|
||||
@@ -246,7 +248,7 @@ template<> EIGEN_STRONG_INLINE Packet2d ei_ploadu<Packet2d>(const double* from)
|
||||
res = _mm_loadh_pd(res,from+1);
|
||||
return res;
|
||||
}
|
||||
template<> EIGEN_STRONG_INLINE Packet4i ei_ploadu<Packet4i>(const int* from)
|
||||
template<> EIGEN_STRONG_INLINE Packet4i ploadu<Packet4i>(const int* from)
|
||||
{
|
||||
EIGEN_DEBUG_UNALIGNED_LOAD
|
||||
__m128d res;
|
||||
@@ -256,71 +258,71 @@ template<> EIGEN_STRONG_INLINE Packet4i ei_ploadu<Packet4i>(const int* from)
|
||||
}
|
||||
#endif
|
||||
|
||||
template<> EIGEN_STRONG_INLINE Packet4f ei_ploaddup<Packet4f>(const float* from)
|
||||
template<> EIGEN_STRONG_INLINE Packet4f ploaddup<Packet4f>(const float* from)
|
||||
{
|
||||
return ei_vec4f_swizzle1(_mm_castpd_ps(_mm_load_sd((const double*)from)), 0, 0, 1, 1);
|
||||
return vec4f_swizzle1(_mm_castpd_ps(_mm_load_sd((const double*)from)), 0, 0, 1, 1);
|
||||
}
|
||||
template<> EIGEN_STRONG_INLINE Packet2d ei_ploaddup<Packet2d>(const double* from)
|
||||
{ return ei_pset1<Packet2d>(from[0]); }
|
||||
template<> EIGEN_STRONG_INLINE Packet4i ei_ploaddup<Packet4i>(const int* from)
|
||||
template<> EIGEN_STRONG_INLINE Packet2d ploaddup<Packet2d>(const double* from)
|
||||
{ return pset1<Packet2d>(from[0]); }
|
||||
template<> EIGEN_STRONG_INLINE Packet4i ploaddup<Packet4i>(const int* from)
|
||||
{
|
||||
Packet4i tmp;
|
||||
tmp = _mm_loadl_epi64(reinterpret_cast<const Packet4i*>(from));
|
||||
return ei_vec4i_swizzle1(tmp, 0, 0, 1, 1);
|
||||
return vec4i_swizzle1(tmp, 0, 0, 1, 1);
|
||||
}
|
||||
|
||||
template<> EIGEN_STRONG_INLINE void ei_pstore<float>(float* to, const Packet4f& from) { EIGEN_DEBUG_ALIGNED_STORE _mm_store_ps(to, from); }
|
||||
template<> EIGEN_STRONG_INLINE void ei_pstore<double>(double* to, const Packet2d& from) { EIGEN_DEBUG_ALIGNED_STORE _mm_store_pd(to, from); }
|
||||
template<> EIGEN_STRONG_INLINE void ei_pstore<int>(int* to, const Packet4i& from) { EIGEN_DEBUG_ALIGNED_STORE _mm_store_si128(reinterpret_cast<Packet4i*>(to), from); }
|
||||
template<> EIGEN_STRONG_INLINE void pstore<float>(float* to, const Packet4f& from) { EIGEN_DEBUG_ALIGNED_STORE _mm_store_ps(to, from); }
|
||||
template<> EIGEN_STRONG_INLINE void pstore<double>(double* to, const Packet2d& from) { EIGEN_DEBUG_ALIGNED_STORE _mm_store_pd(to, from); }
|
||||
template<> EIGEN_STRONG_INLINE void pstore<int>(int* to, const Packet4i& from) { EIGEN_DEBUG_ALIGNED_STORE _mm_store_si128(reinterpret_cast<Packet4i*>(to), from); }
|
||||
|
||||
template<> EIGEN_STRONG_INLINE void ei_pstoreu<double>(double* to, const Packet2d& from) {
|
||||
template<> EIGEN_STRONG_INLINE void pstoreu<double>(double* to, const Packet2d& from) {
|
||||
EIGEN_DEBUG_UNALIGNED_STORE
|
||||
_mm_storel_pd((to), from);
|
||||
_mm_storeh_pd((to+1), from);
|
||||
}
|
||||
template<> EIGEN_STRONG_INLINE void ei_pstoreu<float>(float* to, const Packet4f& from) { EIGEN_DEBUG_UNALIGNED_STORE ei_pstoreu((double*)to, _mm_castps_pd(from)); }
|
||||
template<> EIGEN_STRONG_INLINE void ei_pstoreu<int>(int* to, const Packet4i& from) { EIGEN_DEBUG_UNALIGNED_STORE ei_pstoreu((double*)to, _mm_castsi128_pd(from)); }
|
||||
template<> EIGEN_STRONG_INLINE void pstoreu<float>(float* to, const Packet4f& from) { EIGEN_DEBUG_UNALIGNED_STORE pstoreu((double*)to, _mm_castps_pd(from)); }
|
||||
template<> EIGEN_STRONG_INLINE void pstoreu<int>(int* to, const Packet4i& from) { EIGEN_DEBUG_UNALIGNED_STORE pstoreu((double*)to, _mm_castsi128_pd(from)); }
|
||||
|
||||
template<> EIGEN_STRONG_INLINE void ei_prefetch<float>(const float* addr) { _mm_prefetch((const char*)(addr), _MM_HINT_T0); }
|
||||
template<> EIGEN_STRONG_INLINE void ei_prefetch<double>(const double* addr) { _mm_prefetch((const char*)(addr), _MM_HINT_T0); }
|
||||
template<> EIGEN_STRONG_INLINE void ei_prefetch<int>(const int* addr) { _mm_prefetch((const char*)(addr), _MM_HINT_T0); }
|
||||
template<> EIGEN_STRONG_INLINE void prefetch<float>(const float* addr) { _mm_prefetch((const char*)(addr), _MM_HINT_T0); }
|
||||
template<> EIGEN_STRONG_INLINE void prefetch<double>(const double* addr) { _mm_prefetch((const char*)(addr), _MM_HINT_T0); }
|
||||
template<> EIGEN_STRONG_INLINE void prefetch<int>(const int* addr) { _mm_prefetch((const char*)(addr), _MM_HINT_T0); }
|
||||
|
||||
#if defined(_MSC_VER) && (_MSC_VER <= 1500) && defined(_WIN64) && !defined(__INTEL_COMPILER)
|
||||
// The temporary variable fixes an internal compilation error.
|
||||
// Direct of the struct members fixed bug #62.
|
||||
template<> EIGEN_STRONG_INLINE float ei_pfirst<Packet4f>(const Packet4f& a) { return a.m128_f32[0]; }
|
||||
template<> EIGEN_STRONG_INLINE double ei_pfirst<Packet2d>(const Packet2d& a) { return a.m128d_f64[0]; }
|
||||
template<> EIGEN_STRONG_INLINE int ei_pfirst<Packet4i>(const Packet4i& a) { int x = _mm_cvtsi128_si32(a); return x; }
|
||||
template<> EIGEN_STRONG_INLINE float pfirst<Packet4f>(const Packet4f& a) { return a.m128_f32[0]; }
|
||||
template<> EIGEN_STRONG_INLINE double pfirst<Packet2d>(const Packet2d& a) { return a.m128d_f64[0]; }
|
||||
template<> EIGEN_STRONG_INLINE int pfirst<Packet4i>(const Packet4i& a) { int x = _mm_cvtsi128_si32(a); return x; }
|
||||
#elif defined(_MSC_VER) && (_MSC_VER <= 1500) && !defined(__INTEL_COMPILER)
|
||||
// The temporary variable fixes an internal compilation error.
|
||||
template<> EIGEN_STRONG_INLINE float ei_pfirst<Packet4f>(const Packet4f& a) { float x = _mm_cvtss_f32(a); return x; }
|
||||
template<> EIGEN_STRONG_INLINE double ei_pfirst<Packet2d>(const Packet2d& a) { double x = _mm_cvtsd_f64(a); return x; }
|
||||
template<> EIGEN_STRONG_INLINE int ei_pfirst<Packet4i>(const Packet4i& a) { int x = _mm_cvtsi128_si32(a); return x; }
|
||||
template<> EIGEN_STRONG_INLINE float pfirst<Packet4f>(const Packet4f& a) { float x = _mm_cvtss_f32(a); return x; }
|
||||
template<> EIGEN_STRONG_INLINE double pfirst<Packet2d>(const Packet2d& a) { double x = _mm_cvtsd_f64(a); return x; }
|
||||
template<> EIGEN_STRONG_INLINE int pfirst<Packet4i>(const Packet4i& a) { int x = _mm_cvtsi128_si32(a); return x; }
|
||||
#else
|
||||
template<> EIGEN_STRONG_INLINE float ei_pfirst<Packet4f>(const Packet4f& a) { return _mm_cvtss_f32(a); }
|
||||
template<> EIGEN_STRONG_INLINE double ei_pfirst<Packet2d>(const Packet2d& a) { return _mm_cvtsd_f64(a); }
|
||||
template<> EIGEN_STRONG_INLINE int ei_pfirst<Packet4i>(const Packet4i& a) { return _mm_cvtsi128_si32(a); }
|
||||
template<> EIGEN_STRONG_INLINE float pfirst<Packet4f>(const Packet4f& a) { return _mm_cvtss_f32(a); }
|
||||
template<> EIGEN_STRONG_INLINE double pfirst<Packet2d>(const Packet2d& a) { return _mm_cvtsd_f64(a); }
|
||||
template<> EIGEN_STRONG_INLINE int pfirst<Packet4i>(const Packet4i& a) { return _mm_cvtsi128_si32(a); }
|
||||
#endif
|
||||
|
||||
template<> EIGEN_STRONG_INLINE Packet4f ei_preverse(const Packet4f& a)
|
||||
template<> EIGEN_STRONG_INLINE Packet4f preverse(const Packet4f& a)
|
||||
{ return _mm_shuffle_ps(a,a,0x1B); }
|
||||
template<> EIGEN_STRONG_INLINE Packet2d ei_preverse(const Packet2d& a)
|
||||
template<> EIGEN_STRONG_INLINE Packet2d preverse(const Packet2d& a)
|
||||
{ return _mm_shuffle_pd(a,a,0x1); }
|
||||
template<> EIGEN_STRONG_INLINE Packet4i ei_preverse(const Packet4i& a)
|
||||
template<> EIGEN_STRONG_INLINE Packet4i preverse(const Packet4i& a)
|
||||
{ return _mm_shuffle_epi32(a,0x1B); }
|
||||
|
||||
|
||||
template<> EIGEN_STRONG_INLINE Packet4f ei_pabs(const Packet4f& a)
|
||||
template<> EIGEN_STRONG_INLINE Packet4f pabs(const Packet4f& a)
|
||||
{
|
||||
const Packet4f mask = _mm_castsi128_ps(_mm_setr_epi32(0x7FFFFFFF,0x7FFFFFFF,0x7FFFFFFF,0x7FFFFFFF));
|
||||
return _mm_and_ps(a,mask);
|
||||
}
|
||||
template<> EIGEN_STRONG_INLINE Packet2d ei_pabs(const Packet2d& a)
|
||||
template<> EIGEN_STRONG_INLINE Packet2d pabs(const Packet2d& a)
|
||||
{
|
||||
const Packet2d mask = _mm_castsi128_pd(_mm_setr_epi32(0xFFFFFFFF,0x7FFFFFFF,0xFFFFFFFF,0x7FFFFFFF));
|
||||
return _mm_and_pd(a,mask);
|
||||
}
|
||||
template<> EIGEN_STRONG_INLINE Packet4i ei_pabs(const Packet4i& a)
|
||||
template<> EIGEN_STRONG_INLINE Packet4i pabs(const Packet4i& a)
|
||||
{
|
||||
#ifdef EIGEN_VECTORIZE_SSSE3
|
||||
return _mm_abs_epi32(a);
|
||||
@@ -330,7 +332,7 @@ template<> EIGEN_STRONG_INLINE Packet4i ei_pabs(const Packet4i& a)
|
||||
#endif
|
||||
}
|
||||
|
||||
EIGEN_STRONG_INLINE void ei_punpackp(Packet4f* vecs)
|
||||
EIGEN_STRONG_INLINE void punpackp(Packet4f* vecs)
|
||||
{
|
||||
vecs[1] = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(vecs[0]), 0x55));
|
||||
vecs[2] = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(vecs[0]), 0xAA));
|
||||
@@ -340,47 +342,47 @@ EIGEN_STRONG_INLINE void ei_punpackp(Packet4f* vecs)
|
||||
|
||||
#ifdef EIGEN_VECTORIZE_SSE3
|
||||
// TODO implement SSE2 versions as well as integer versions
|
||||
template<> EIGEN_STRONG_INLINE Packet4f ei_preduxp<Packet4f>(const Packet4f* vecs)
|
||||
template<> EIGEN_STRONG_INLINE Packet4f preduxp<Packet4f>(const Packet4f* vecs)
|
||||
{
|
||||
return _mm_hadd_ps(_mm_hadd_ps(vecs[0], vecs[1]),_mm_hadd_ps(vecs[2], vecs[3]));
|
||||
}
|
||||
template<> EIGEN_STRONG_INLINE Packet2d ei_preduxp<Packet2d>(const Packet2d* vecs)
|
||||
template<> EIGEN_STRONG_INLINE Packet2d preduxp<Packet2d>(const Packet2d* vecs)
|
||||
{
|
||||
return _mm_hadd_pd(vecs[0], vecs[1]);
|
||||
}
|
||||
// SSSE3 version:
|
||||
// EIGEN_STRONG_INLINE Packet4i ei_preduxp(const Packet4i* vecs)
|
||||
// EIGEN_STRONG_INLINE Packet4i preduxp(const Packet4i* vecs)
|
||||
// {
|
||||
// return _mm_hadd_epi32(_mm_hadd_epi32(vecs[0], vecs[1]),_mm_hadd_epi32(vecs[2], vecs[3]));
|
||||
// }
|
||||
|
||||
template<> EIGEN_STRONG_INLINE float ei_predux<Packet4f>(const Packet4f& a)
|
||||
template<> EIGEN_STRONG_INLINE float predux<Packet4f>(const Packet4f& a)
|
||||
{
|
||||
Packet4f tmp0 = _mm_hadd_ps(a,a);
|
||||
return ei_pfirst(_mm_hadd_ps(tmp0, tmp0));
|
||||
return pfirst(_mm_hadd_ps(tmp0, tmp0));
|
||||
}
|
||||
|
||||
template<> EIGEN_STRONG_INLINE double ei_predux<Packet2d>(const Packet2d& a) { return ei_pfirst(_mm_hadd_pd(a, a)); }
|
||||
template<> EIGEN_STRONG_INLINE double predux<Packet2d>(const Packet2d& a) { return pfirst(_mm_hadd_pd(a, a)); }
|
||||
|
||||
// SSSE3 version:
|
||||
// EIGEN_STRONG_INLINE float ei_predux(const Packet4i& a)
|
||||
// EIGEN_STRONG_INLINE float predux(const Packet4i& a)
|
||||
// {
|
||||
// Packet4i tmp0 = _mm_hadd_epi32(a,a);
|
||||
// return ei_pfirst(_mm_hadd_epi32(tmp0, tmp0));
|
||||
// return pfirst(_mm_hadd_epi32(tmp0, tmp0));
|
||||
// }
|
||||
#else
|
||||
// SSE2 versions
|
||||
template<> EIGEN_STRONG_INLINE float ei_predux<Packet4f>(const Packet4f& a)
|
||||
template<> EIGEN_STRONG_INLINE float predux<Packet4f>(const Packet4f& a)
|
||||
{
|
||||
Packet4f tmp = _mm_add_ps(a, _mm_movehl_ps(a,a));
|
||||
return ei_pfirst(_mm_add_ss(tmp, _mm_shuffle_ps(tmp,tmp, 1)));
|
||||
return pfirst(_mm_add_ss(tmp, _mm_shuffle_ps(tmp,tmp, 1)));
|
||||
}
|
||||
template<> EIGEN_STRONG_INLINE double ei_predux<Packet2d>(const Packet2d& a)
|
||||
template<> EIGEN_STRONG_INLINE double predux<Packet2d>(const Packet2d& a)
|
||||
{
|
||||
return ei_pfirst(_mm_add_sd(a, _mm_unpackhi_pd(a,a)));
|
||||
return pfirst(_mm_add_sd(a, _mm_unpackhi_pd(a,a)));
|
||||
}
|
||||
|
||||
template<> EIGEN_STRONG_INLINE Packet4f ei_preduxp<Packet4f>(const Packet4f* vecs)
|
||||
template<> EIGEN_STRONG_INLINE Packet4f preduxp<Packet4f>(const Packet4f* vecs)
|
||||
{
|
||||
Packet4f tmp0, tmp1, tmp2;
|
||||
tmp0 = _mm_unpacklo_ps(vecs[0], vecs[1]);
|
||||
@@ -394,19 +396,19 @@ template<> EIGEN_STRONG_INLINE Packet4f ei_preduxp<Packet4f>(const Packet4f* vec
|
||||
return _mm_add_ps(tmp0, tmp2);
|
||||
}
|
||||
|
||||
template<> EIGEN_STRONG_INLINE Packet2d ei_preduxp<Packet2d>(const Packet2d* vecs)
|
||||
template<> EIGEN_STRONG_INLINE Packet2d preduxp<Packet2d>(const Packet2d* vecs)
|
||||
{
|
||||
return _mm_add_pd(_mm_unpacklo_pd(vecs[0], vecs[1]), _mm_unpackhi_pd(vecs[0], vecs[1]));
|
||||
}
|
||||
#endif // SSE3
|
||||
|
||||
template<> EIGEN_STRONG_INLINE int ei_predux<Packet4i>(const Packet4i& a)
|
||||
template<> EIGEN_STRONG_INLINE int predux<Packet4i>(const Packet4i& a)
|
||||
{
|
||||
Packet4i tmp = _mm_add_epi32(a, _mm_unpackhi_epi64(a,a));
|
||||
return ei_pfirst(tmp) + ei_pfirst(_mm_shuffle_epi32(tmp, 1));
|
||||
return pfirst(tmp) + pfirst(_mm_shuffle_epi32(tmp, 1));
|
||||
}
|
||||
|
||||
template<> EIGEN_STRONG_INLINE Packet4i ei_preduxp<Packet4i>(const Packet4i* vecs)
|
||||
template<> EIGEN_STRONG_INLINE Packet4i preduxp<Packet4i>(const Packet4i* vecs)
|
||||
{
|
||||
Packet4i tmp0, tmp1, tmp2;
|
||||
tmp0 = _mm_unpacklo_epi32(vecs[0], vecs[1]);
|
||||
@@ -423,69 +425,69 @@ template<> EIGEN_STRONG_INLINE Packet4i ei_preduxp<Packet4i>(const Packet4i* vec
|
||||
// Other reduction functions:
|
||||
|
||||
// mul
|
||||
template<> EIGEN_STRONG_INLINE float ei_predux_mul<Packet4f>(const Packet4f& a)
|
||||
template<> EIGEN_STRONG_INLINE float predux_mul<Packet4f>(const Packet4f& a)
|
||||
{
|
||||
Packet4f tmp = _mm_mul_ps(a, _mm_movehl_ps(a,a));
|
||||
return ei_pfirst(_mm_mul_ss(tmp, _mm_shuffle_ps(tmp,tmp, 1)));
|
||||
return pfirst(_mm_mul_ss(tmp, _mm_shuffle_ps(tmp,tmp, 1)));
|
||||
}
|
||||
template<> EIGEN_STRONG_INLINE double ei_predux_mul<Packet2d>(const Packet2d& a)
|
||||
template<> EIGEN_STRONG_INLINE double predux_mul<Packet2d>(const Packet2d& a)
|
||||
{
|
||||
return ei_pfirst(_mm_mul_sd(a, _mm_unpackhi_pd(a,a)));
|
||||
return pfirst(_mm_mul_sd(a, _mm_unpackhi_pd(a,a)));
|
||||
}
|
||||
template<> EIGEN_STRONG_INLINE int ei_predux_mul<Packet4i>(const Packet4i& a)
|
||||
template<> EIGEN_STRONG_INLINE int predux_mul<Packet4i>(const Packet4i& a)
|
||||
{
|
||||
// after some experiments, it is seems this is the fastest way to implement it
|
||||
// for GCC (eg., reusing ei_pmul is very slow !)
|
||||
// for GCC (eg., reusing pmul is very slow !)
|
||||
// TODO try to call _mm_mul_epu32 directly
|
||||
EIGEN_ALIGN16 int aux[4];
|
||||
ei_pstore(aux, a);
|
||||
pstore(aux, a);
|
||||
return (aux[0] * aux[1]) * (aux[2] * aux[3]);;
|
||||
}
|
||||
|
||||
// min
|
||||
template<> EIGEN_STRONG_INLINE float ei_predux_min<Packet4f>(const Packet4f& a)
|
||||
template<> EIGEN_STRONG_INLINE float predux_min<Packet4f>(const Packet4f& a)
|
||||
{
|
||||
Packet4f tmp = _mm_min_ps(a, _mm_movehl_ps(a,a));
|
||||
return ei_pfirst(_mm_min_ss(tmp, _mm_shuffle_ps(tmp,tmp, 1)));
|
||||
return pfirst(_mm_min_ss(tmp, _mm_shuffle_ps(tmp,tmp, 1)));
|
||||
}
|
||||
template<> EIGEN_STRONG_INLINE double ei_predux_min<Packet2d>(const Packet2d& a)
|
||||
template<> EIGEN_STRONG_INLINE double predux_min<Packet2d>(const Packet2d& a)
|
||||
{
|
||||
return ei_pfirst(_mm_min_sd(a, _mm_unpackhi_pd(a,a)));
|
||||
return pfirst(_mm_min_sd(a, _mm_unpackhi_pd(a,a)));
|
||||
}
|
||||
template<> EIGEN_STRONG_INLINE int ei_predux_min<Packet4i>(const Packet4i& a)
|
||||
template<> EIGEN_STRONG_INLINE int predux_min<Packet4i>(const Packet4i& a)
|
||||
{
|
||||
// after some experiments, it is seems this is the fastest way to implement it
|
||||
// for GCC (eg., it does not like using std::min after the ei_pstore !!)
|
||||
// for GCC (eg., it does not like using std::min after the pstore !!)
|
||||
EIGEN_ALIGN16 int aux[4];
|
||||
ei_pstore(aux, a);
|
||||
pstore(aux, a);
|
||||
register int aux0 = aux[0]<aux[1] ? aux[0] : aux[1];
|
||||
register int aux2 = aux[2]<aux[3] ? aux[2] : aux[3];
|
||||
return aux0<aux2 ? aux0 : aux2;
|
||||
}
|
||||
|
||||
// max
|
||||
template<> EIGEN_STRONG_INLINE float ei_predux_max<Packet4f>(const Packet4f& a)
|
||||
template<> EIGEN_STRONG_INLINE float predux_max<Packet4f>(const Packet4f& a)
|
||||
{
|
||||
Packet4f tmp = _mm_max_ps(a, _mm_movehl_ps(a,a));
|
||||
return ei_pfirst(_mm_max_ss(tmp, _mm_shuffle_ps(tmp,tmp, 1)));
|
||||
return pfirst(_mm_max_ss(tmp, _mm_shuffle_ps(tmp,tmp, 1)));
|
||||
}
|
||||
template<> EIGEN_STRONG_INLINE double ei_predux_max<Packet2d>(const Packet2d& a)
|
||||
template<> EIGEN_STRONG_INLINE double predux_max<Packet2d>(const Packet2d& a)
|
||||
{
|
||||
return ei_pfirst(_mm_max_sd(a, _mm_unpackhi_pd(a,a)));
|
||||
return pfirst(_mm_max_sd(a, _mm_unpackhi_pd(a,a)));
|
||||
}
|
||||
template<> EIGEN_STRONG_INLINE int ei_predux_max<Packet4i>(const Packet4i& a)
|
||||
template<> EIGEN_STRONG_INLINE int predux_max<Packet4i>(const Packet4i& a)
|
||||
{
|
||||
// after some experiments, it is seems this is the fastest way to implement it
|
||||
// for GCC (eg., it does not like using std::min after the ei_pstore !!)
|
||||
// for GCC (eg., it does not like using std::min after the pstore !!)
|
||||
EIGEN_ALIGN16 int aux[4];
|
||||
ei_pstore(aux, a);
|
||||
pstore(aux, a);
|
||||
register int aux0 = aux[0]>aux[1] ? aux[0] : aux[1];
|
||||
register int aux2 = aux[2]>aux[3] ? aux[2] : aux[3];
|
||||
return aux0>aux2 ? aux0 : aux2;
|
||||
}
|
||||
|
||||
#if (defined __GNUC__)
|
||||
// template <> EIGEN_STRONG_INLINE Packet4f ei_pmadd(const Packet4f& a, const Packet4f& b, const Packet4f& c)
|
||||
// template <> EIGEN_STRONG_INLINE Packet4f pmadd(const Packet4f& a, const Packet4f& b, const Packet4f& c)
|
||||
// {
|
||||
// Packet4f res = b;
|
||||
// asm("mulps %[a], %[b] \n\taddps %[c], %[b]" : [b] "+x" (res) : [a] "x" (a), [c] "x" (c));
|
||||
@@ -502,7 +504,7 @@ template<> EIGEN_STRONG_INLINE int ei_predux_max<Packet4i>(const Packet4i& a)
|
||||
#ifdef EIGEN_VECTORIZE_SSSE3
|
||||
// SSSE3 versions
|
||||
template<int Offset>
|
||||
struct ei_palign_impl<Offset,Packet4f>
|
||||
struct palign_impl<Offset,Packet4f>
|
||||
{
|
||||
EIGEN_STRONG_INLINE static void run(Packet4f& first, const Packet4f& second)
|
||||
{
|
||||
@@ -512,7 +514,7 @@ struct ei_palign_impl<Offset,Packet4f>
|
||||
};
|
||||
|
||||
template<int Offset>
|
||||
struct ei_palign_impl<Offset,Packet4i>
|
||||
struct palign_impl<Offset,Packet4i>
|
||||
{
|
||||
EIGEN_STRONG_INLINE static void run(Packet4i& first, const Packet4i& second)
|
||||
{
|
||||
@@ -522,7 +524,7 @@ struct ei_palign_impl<Offset,Packet4i>
|
||||
};
|
||||
|
||||
template<int Offset>
|
||||
struct ei_palign_impl<Offset,Packet2d>
|
||||
struct palign_impl<Offset,Packet2d>
|
||||
{
|
||||
EIGEN_STRONG_INLINE static void run(Packet2d& first, const Packet2d& second)
|
||||
{
|
||||
@@ -533,7 +535,7 @@ struct ei_palign_impl<Offset,Packet2d>
|
||||
#else
|
||||
// SSE2 versions
|
||||
template<int Offset>
|
||||
struct ei_palign_impl<Offset,Packet4f>
|
||||
struct palign_impl<Offset,Packet4f>
|
||||
{
|
||||
EIGEN_STRONG_INLINE static void run(Packet4f& first, const Packet4f& second)
|
||||
{
|
||||
@@ -556,7 +558,7 @@ struct ei_palign_impl<Offset,Packet4f>
|
||||
};
|
||||
|
||||
template<int Offset>
|
||||
struct ei_palign_impl<Offset,Packet4i>
|
||||
struct palign_impl<Offset,Packet4i>
|
||||
{
|
||||
EIGEN_STRONG_INLINE static void run(Packet4i& first, const Packet4i& second)
|
||||
{
|
||||
@@ -579,7 +581,7 @@ struct ei_palign_impl<Offset,Packet4i>
|
||||
};
|
||||
|
||||
template<int Offset>
|
||||
struct ei_palign_impl<Offset,Packet2d>
|
||||
struct palign_impl<Offset,Packet2d>
|
||||
{
|
||||
EIGEN_STRONG_INLINE static void run(Packet2d& first, const Packet2d& second)
|
||||
{
|
||||
@@ -592,4 +594,6 @@ struct ei_palign_impl<Offset,Packet2d>
|
||||
};
|
||||
#endif
|
||||
|
||||
} // end namespace internal
|
||||
|
||||
#endif // EIGEN_PACKET_MATH_SSE_H
|
||||
|
||||
Reference in New Issue
Block a user