devtools/eigen
1
0
Fork 0
mirror of https://gitlab.com/libeigen/eigen.git synced 2026-04-10 11:34:33 +08:00
Code Issues Packages Projects Releases Wiki Activity

Add truncation op

Browse Source
This commit is contained in:
Charles Schlosser
2024-04-29 23:45:49 +00:00
committed by Rasmus Munk Larsen
parent d5524fc57b
commit fb95e90f7f
23 changed files with 333 additions and 267 deletions
Download Patch File Download Diff File Expand all files Collapse all files

39
Eigen/src/Core/arch/AVX/PacketMath.h
View File

@@ -124,11 +124,7 @@ struct packet_traits<float> : default_packet_traits {
HasRsqrt = 1,
HasTanh = EIGEN_FAST_MATH,
HasErf = EIGEN_FAST_MATH,
HasBlend = 1,
HasRound = 1,
HasFloor = 1,
HasCeil = 1,
HasRint = 1
HasBlend = 1
};
};
template <>
@@ -151,11 +147,7 @@ struct packet_traits<double> : default_packet_traits {
HasSqrt = 1,
HasRsqrt = 1,
HasATan = 1,
HasBlend = 1,
HasRound = 1,
HasFloor = 1,
HasCeil = 1,
HasRint = 1
HasBlend = 1
};
};
@@ -192,10 +184,6 @@ struct packet_traits<Eigen::half> : default_packet_traits {
HasTanh = EIGEN_FAST_MATH,
HasErf = EIGEN_FAST_MATH,
HasBlend = 0,
HasRound = 1,
HasFloor = 1,
HasCeil = 1,
HasRint = 1,
HasBessel = 1,
HasNdtri = 1
};
@@ -235,10 +223,6 @@ struct packet_traits<bfloat16> : default_packet_traits {
HasTanh = EIGEN_FAST_MATH,
HasErf = EIGEN_FAST_MATH,
HasBlend = 0,
HasRound = 1,
HasFloor = 1,
HasCeil = 1,
HasRint = 1,
HasBessel = 1,
HasNdtri = 1
};
@@ -1257,6 +1241,15 @@ EIGEN_STRONG_INLINE Packet4d pfloor<Packet4d>(const Packet4d& a) {
return _mm256_floor_pd(a);
}
template <>
EIGEN_STRONG_INLINE Packet8f ptrunc<Packet8f>(const Packet8f& a) {
return _mm256_round_ps(a, _MM_FROUND_TRUNC);
}
template <>
EIGEN_STRONG_INLINE Packet4d ptrunc<Packet4d>(const Packet4d& a) {
return _mm256_round_pd(a, _MM_FROUND_TRUNC);
}
template <>
EIGEN_STRONG_INLINE Packet8i ptrue<Packet8i>(const Packet8i& a) {
#ifdef EIGEN_VECTORIZE_AVX2
@@ -2311,6 +2304,11 @@ EIGEN_STRONG_INLINE Packet8h pfloor<Packet8h>(const Packet8h& a) {
return float2half(pfloor<Packet8f>(half2float(a)));
}
template <>
EIGEN_STRONG_INLINE Packet8h ptrunc<Packet8h>(const Packet8h& a) {
return float2half(ptrunc<Packet8f>(half2float(a)));
}
template <>
EIGEN_STRONG_INLINE Packet8h pcmp_eq(const Packet8h& a, const Packet8h& b) {
return Pack16To8(pcmp_eq(half2float(a), half2float(b)));
@@ -2686,6 +2684,11 @@ EIGEN_STRONG_INLINE Packet8bf pfloor<Packet8bf>(const Packet8bf& a) {
return F32ToBf16(pfloor<Packet8f>(Bf16ToF32(a)));
}
template <>
EIGEN_STRONG_INLINE Packet8bf ptrunc<Packet8bf>(const Packet8bf& a) {
return F32ToBf16(ptrunc<Packet8f>(Bf16ToF32(a)));
}
template <>
EIGEN_STRONG_INLINE Packet8bf pcmp_eq(const Packet8bf& a, const Packet8bf& b) {
return Pack16To8(pcmp_eq(Bf16ToF32(a), Bf16ToF32(b)));

37
Eigen/src/Core/arch/AVX512/PacketMath.h
View File

@@ -97,11 +97,7 @@ struct packet_traits<half> : default_packet_traits {
HasCos = EIGEN_FAST_MATH,
HasTanh = EIGEN_FAST_MATH,
HasErf = EIGEN_FAST_MATH,
HasBlend = 0,
HasRound = 1,
HasFloor = 1,
HasCeil = 1,
HasRint = 1
HasBlend = 0
};
};
#endif
@@ -138,11 +134,7 @@ struct packet_traits<float> : default_packet_traits {
HasTanh = EIGEN_FAST_MATH,
HasErf = EIGEN_FAST_MATH,
HasCmp = 1,
HasDiv = 1,
HasRound = 1,
HasFloor = 1,
HasCeil = 1,
HasRint = 1
HasDiv = 1
};
};
template <>
@@ -162,11 +154,7 @@ struct packet_traits<double> : default_packet_traits {
HasExp = 1,
HasATan = 1,
HasCmp = 1,
HasDiv = 1,
HasRound = 1,
HasFloor = 1,
HasCeil = 1,
HasRint = 1
HasDiv = 1
};
};
@@ -781,6 +769,15 @@ EIGEN_STRONG_INLINE Packet8d pfloor<Packet8d>(const Packet8d& a) {
return _mm512_roundscale_pd(a, _MM_FROUND_TO_NEG_INF);
}
template <>
EIGEN_STRONG_INLINE Packet16f ptrunc<Packet16f>(const Packet16f& a) {
return _mm512_roundscale_ps(a, _MM_FROUND_TO_ZERO);
}
template <>
EIGEN_STRONG_INLINE Packet8d ptrunc<Packet8d>(const Packet8d& a) {
return _mm512_roundscale_pd(a, _MM_FROUND_TO_ZERO);
}
template <>
EIGEN_STRONG_INLINE Packet16i ptrue<Packet16i>(const Packet16i& /*a*/) {
return _mm512_set1_epi32(int32_t(-1));
@@ -2322,6 +2319,11 @@ EIGEN_STRONG_INLINE Packet16h pfloor<Packet16h>(const Packet16h& a) {
return float2half(pfloor<Packet16f>(half2float(a)));
}
template <>
EIGEN_STRONG_INLINE Packet16h ptrunc<Packet16h>(const Packet16h& a) {
return float2half(ptrunc<Packet16f>(half2float(a)));
}
template <>
EIGEN_STRONG_INLINE Packet16h pcmp_eq(const Packet16h& a, const Packet16h& b) {
Packet16f af = half2float(a);
@@ -2821,6 +2823,11 @@ EIGEN_STRONG_INLINE Packet16bf pfloor<Packet16bf>(const Packet16bf& a) {
return F32ToBf16(pfloor<Packet16f>(Bf16ToF32(a)));
}
template <>
EIGEN_STRONG_INLINE Packet16bf ptrunc<Packet16bf>(const Packet16bf& a) {
return F32ToBf16(ptrunc<Packet16f>(Bf16ToF32(a)));
}
template <>
EIGEN_STRONG_INLINE Packet16bf pcmp_eq(const Packet16bf& a, const Packet16bf& b) {
return Pack32To16(pcmp_eq(Bf16ToF32(a), Bf16ToF32(b)));

13
Eigen/src/Core/arch/AVX512/PacketMathFP16.h
View File

@@ -60,11 +60,7 @@ struct packet_traits<half> : default_packet_traits {
HasCos = EIGEN_FAST_MATH,
HasTanh = EIGEN_FAST_MATH,
HasErf = 0, // EIGEN_FAST_MATH,
HasBlend = 0,
HasRound = 1,
HasFloor = 1,
HasCeil = 1,
HasRint = 1
HasBlend = 0
};
};
@@ -390,6 +386,13 @@ EIGEN_STRONG_INLINE Packet32h pfloor<Packet32h>(const Packet32h& a) {
return _mm512_roundscale_ph(a, _MM_FROUND_TO_NEG_INF);
}
// ptrunc
template <>
EIGEN_STRONG_INLINE Packet32h ptrunc<Packet32h>(const Packet32h& a) {
return _mm512_roundscale_ph(a, _MM_FROUND_TO_ZERO);
}
// predux
template <>
EIGEN_STRONG_INLINE half predux<Packet32h>(const Packet32h& a) {

26
Eigen/src/Core/arch/AltiVec/PacketMath.h
View File

@@ -193,17 +193,12 @@ struct packet_traits<float> : default_packet_traits {
#endif
HasTanh = EIGEN_FAST_MATH,
HasErf = EIGEN_FAST_MATH,
HasRint = 1,
#else
HasSqrt = 0,
HasRsqrt = 0,
HasTanh = 0,
HasErf = 0,
HasRint = 0,
#endif
HasRound = 1,
HasFloor = 1,
HasCeil = 1,
HasNegate = 1,
HasBlend = 1
};
@@ -235,17 +230,12 @@ struct packet_traits<bfloat16> : default_packet_traits {
#else
HasRsqrt = 0,
#endif
HasRint = 1,
#else
HasSqrt = 0,
HasRsqrt = 0,
HasRint = 0,
#endif
HasTanh = 0,
HasErf = 0,
HasRound = 1,
HasFloor = 1,
HasCeil = 1,
HasNegate = 1,
HasBlend = 1
};
@@ -1506,6 +1496,10 @@ template <>
EIGEN_STRONG_INLINE Packet4f pfloor<Packet4f>(const Packet4f& a) {
return vec_floor(a);
}
template <>
EIGEN_STRONG_INLINE Packet4f ptrunc<Packet4f>(const Packet4f& a) {
return vec_trunc(a);
}
#ifdef EIGEN_VECTORIZE_VSX
template <>
EIGEN_STRONG_INLINE Packet4f print<Packet4f>(const Packet4f& a) {
@@ -2364,6 +2358,10 @@ template <>
EIGEN_STRONG_INLINE Packet8bf pround<Packet8bf>(const Packet8bf& a) {
BF16_TO_F32_UNARY_OP_WRAPPER(pround<Packet4f>, a);
}
template <>
EIGEN_STRONG_INLINE Packet8bf ptrunc<Packet8bf>(const Packet8bf& a) {
BF16_TO_F32_UNARY_OP_WRAPPER(ptrunc<Packet4f>, a);
}
#ifdef EIGEN_VECTORIZE_VSX
template <>
EIGEN_STRONG_INLINE Packet8bf print<Packet8bf>(const Packet8bf& a) {
@@ -3189,10 +3187,6 @@ struct packet_traits<double> : default_packet_traits {
#else
HasRsqrt = 0,
#endif
HasRound = 1,
HasFloor = 1,
HasCeil = 1,
HasRint = 1,
HasNegate = 1,
HasBlend = 1
};
@@ -3446,6 +3440,10 @@ EIGEN_STRONG_INLINE Packet2d pfloor<Packet2d>(const Packet2d& a) {
return vec_floor(a);
}
template <>
EIGEN_STRONG_INLINE Packet2d ptrunc<Packet2d>(const Packet2d& a) {
return vec_trunc(a);
}
template <>
EIGEN_STRONG_INLINE Packet2d print<Packet2d>(const Packet2d& a) {
Packet2d res;

1
Eigen/src/Core/arch/Default/BFloat16.h
View File

@@ -637,6 +637,7 @@ EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 floor(const bfloat16& a) { return
EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 ceil(const bfloat16& a) { return bfloat16(::ceilf(float(a))); }
EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 rint(const bfloat16& a) { return bfloat16(::rintf(float(a))); }
EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 round(const bfloat16& a) { return bfloat16(::roundf(float(a))); }
EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 trunc(const bfloat16& a) { return bfloat16(::truncf(float(a))); }
EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 fmod(const bfloat16& a, const bfloat16& b) {
return bfloat16(::fmodf(float(a), float(b)));
}

89
Eigen/src/Core/arch/Default/GenericPacketMathFunctions.h
View File

@@ -2469,6 +2469,95 @@ struct unary_pow_impl<Packet, ScalarExponent, true, true, false> {
}
};
template <typename Packet>
EIGEN_STRONG_INLINE Packet generic_rint(const Packet& a) {
using Scalar = typename unpacket_traits<Packet>::type;
using IntType = typename numext::get_integer_by_size<sizeof(Scalar)>::signed_type;
// Adds and subtracts signum(a) * 2^kMantissaBits to force rounding.
const IntType kLimit = IntType(1) << (NumTraits<Scalar>::digits() - 1);
const Packet cst_limit = pset1<Packet>(static_cast<Scalar>(kLimit));
Packet abs_a = pabs(a);
Packet sign_a = pandnot(a, abs_a);
Packet rint_a = padd(abs_a, cst_limit);
// Don't compile-away addition and subtraction.
EIGEN_OPTIMIZATION_BARRIER(rint_a);
rint_a = psub(rint_a, cst_limit);
rint_a = por(rint_a, sign_a);
// If greater than limit (or NaN), simply return a.
Packet mask = pcmp_lt(abs_a, cst_limit);
Packet result = pselect(mask, rint_a, a);
return result;
}
template <typename Packet>
EIGEN_STRONG_INLINE Packet generic_floor(const Packet& a) {
using Scalar = typename unpacket_traits<Packet>::type;
const Packet cst_1 = pset1<Packet>(Scalar(1));
Packet rint_a = generic_rint(a);
// if a < rint(a), then rint(a) == ceil(a)
Packet mask = pcmp_lt(a, rint_a);
Packet offset = pand(cst_1, mask);
Packet result = psub(rint_a, offset);
return result;
}
template <typename Packet>
EIGEN_STRONG_INLINE Packet generic_ceil(const Packet& a) {
using Scalar = typename unpacket_traits<Packet>::type;
const Packet cst_1 = pset1<Packet>(Scalar(1));
Packet rint_a = generic_rint(a);
// if rint(a) < a, then rint(a) == floor(a)
Packet mask = pcmp_lt(rint_a, a);
Packet offset = pand(cst_1, mask);
Packet result = padd(rint_a, offset);
return result;
}
template <typename Packet>
EIGEN_STRONG_INLINE Packet generic_trunc(const Packet& a) {
Packet abs_a = pabs(a);
Packet sign_a = pandnot(a, abs_a);
Packet floor_abs_a = generic_floor(abs_a);
Packet result = por(floor_abs_a, sign_a);
return result;
}
template <typename Packet>
EIGEN_STRONG_INLINE Packet generic_round(const Packet& a) {
using Scalar = typename unpacket_traits<Packet>::type;
const Packet cst_half = pset1<Packet>(Scalar(0.5));
const Packet cst_1 = pset1<Packet>(Scalar(1));
Packet abs_a = pabs(a);
Packet sign_a = pandnot(a, abs_a);
Packet floor_abs_a = generic_floor(abs_a);
Packet diff = psub(abs_a, floor_abs_a);
Packet mask = pcmp_le(cst_half, diff);
Packet offset = pand(cst_1, mask);
Packet result = padd(floor_abs_a, offset);
result = por(result, sign_a);
return result;
}
template <typename Packet>
struct nearest_integer_packetop_impl<Packet, /*IsScalar*/ false, /*IsInteger*/ false> {
using Scalar = typename unpacket_traits<Packet>::type;
static_assert(packet_traits<Scalar>::HasRound, "Generic nearest integer functions are disabled for this type.");
static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet run_floor(const Packet& x) { return generic_floor(x); }
static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet run_ceil(const Packet& x) { return generic_ceil(x); }
static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet run_rint(const Packet& x) { return generic_rint(x); }
static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet run_round(const Packet& x) { return generic_round(x); }
static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet run_trunc(const Packet& x) { return generic_trunc(x); }
};
template <typename Packet>
struct nearest_integer_packetop_impl<Packet, /*IsScalar*/ false, /*IsInteger*/ true> {
static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet run_floor(const Packet& x) { return x; }
static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet run_ceil(const Packet& x) { return x; }
static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet run_rint(const Packet& x) { return x; }
static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet run_round(const Packet& x) { return x; }
static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet run_trunc(const Packet& x) { return x; }
};
} // end namespace internal
} // end namespace Eigen

15
Eigen/src/Core/arch/Default/GenericPacketMathFunctionsFwd.h
View File

@@ -133,6 +133,21 @@ EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet plog_complex(const Pa
template <typename Packet>
EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet pexp_complex(const Packet& x);
template <typename Packet>
EIGEN_STRONG_INLINE Packet generic_rint(const Packet& a);
template <typename Packet>
EIGEN_STRONG_INLINE Packet generic_floor(const Packet& a);
template <typename Packet>
EIGEN_STRONG_INLINE Packet generic_ceil(const Packet& a);
template <typename Packet>
EIGEN_STRONG_INLINE Packet generic_trunc(const Packet& a);
template <typename Packet>
EIGEN_STRONG_INLINE Packet generic_round(const Packet& a);
// Macros for instantiating these generic functions for different backends.
#define EIGEN_PACKET_FUNCTION(METHOD, SCALAR, PACKET) \
template <> \

1
Eigen/src/Core/arch/Default/Half.h
View File

@@ -722,6 +722,7 @@ EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half ceil(const half& a) {
}
EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half rint(const half& a) { return half(::rintf(float(a))); }
EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half round(const half& a) { return half(::roundf(float(a))); }
EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half trunc(const half& a) { return half(::truncf(float(a))); }
EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half fmod(const half& a, const half& b) {
return half(::fmodf(float(a), float(b)));
}

3
Eigen/src/Core/arch/GPU/PacketMath.h
View File

@@ -75,8 +75,7 @@ struct packet_traits<float> : default_packet_traits {
HasIGammac = 1,
HasBetaInc = 1,
HasBlend = 0,
HasFloor = 1,
HasBlend = 0
};
};

3
Eigen/src/Core/arch/HVX/PacketMath.h
View File

@@ -161,9 +161,6 @@ struct packet_traits<float> : default_packet_traits {
HasBlend = 0,
HasDiv = 0,
HasFloor = 0,
HasCeil = 0,
HasRint = 0,
HasSin = 0,
HasCos = 0,

6
Eigen/src/Core/arch/MSA/PacketMath.h
View File

@@ -91,9 +91,6 @@ struct packet_traits<float> : default_packet_traits {
HasExp = 1,
HasSqrt = 1,
HasRsqrt = 1,
HasRound = 1,
HasFloor = 1,
HasCeil = 1,
HasBlend = 1
};
};
@@ -859,9 +856,6 @@ struct packet_traits<double> : default_packet_traits {
HasExp = 1,
HasSqrt = 1,
HasRsqrt = 1,
HasRound = 1,
HasFloor = 1,
HasCeil = 1,
HasBlend = 1
};
};

122
Eigen/src/Core/arch/NEON/PacketMath.h
View File

@@ -196,12 +196,7 @@ struct packet_traits<float> : default_packet_traits {
HasConj = 1,
HasSetLinear = 1,
HasBlend = 0,
HasDiv = 1,
HasFloor = 1,
HasCeil = 1,
HasRint = 1,
HasSin = EIGEN_FAST_MATH,
HasCos = EIGEN_FAST_MATH,
HasACos = 1,
@@ -4470,76 +4465,25 @@ EIGEN_STRONG_INLINE Packet4f pceil<Packet4f>(const Packet4f& a) {
return vrndpq_f32(a);
}
#else
template <>
EIGEN_STRONG_INLINE Packet4f print(const Packet4f& a) {
// Adds and subtracts signum(a) * 2^23 to force rounding.
const Packet4f limit = pset1<Packet4f>(static_cast<float>(1 << 23));
const Packet4f abs_a = pabs(a);
Packet4f r = padd(abs_a, limit);
// Don't compile-away addition and subtraction.
EIGEN_OPTIMIZATION_BARRIER(r);
r = psub(r, limit);
// If greater than limit, simply return a. Otherwise, account for sign.
r = pselect(pcmp_lt(abs_a, limit), pselect(pcmp_lt(a, pzero(a)), pnegate(r), r), a);
return r;
EIGEN_STRONG_INLINE Packet2f pround<Packet2f>(const Packet2f& a) {
return vrnda_f32(a);
}
template <>
EIGEN_STRONG_INLINE Packet2f print(const Packet2f& a) {
// Adds and subtracts signum(a) * 2^23 to force rounding.
const Packet2f limit = pset1<Packet2f>(static_cast<float>(1 << 23));
const Packet2f abs_a = pabs(a);
Packet2f r = padd(abs_a, limit);
// Don't compile-away addition and subtraction.
EIGEN_OPTIMIZATION_BARRIER(r);
r = psub(r, limit);
// If greater than limit, simply return a. Otherwise, account for sign.
r = pselect(pcmp_lt(abs_a, limit), pselect(pcmp_lt(a, pzero(a)), pnegate(r), r), a);
return r;
EIGEN_STRONG_INLINE Packet4f pround<Packet4f>(const Packet4f& a) {
return vrndaq_f32(a);
}
template <>
EIGEN_STRONG_INLINE Packet4f pfloor<Packet4f>(const Packet4f& a) {
const Packet4f cst_1 = pset1<Packet4f>(1.0f);
Packet4f tmp = print<Packet4f>(a);
// If greater, subtract one.
Packet4f mask = pcmp_lt(a, tmp);
mask = pand(mask, cst_1);
return psub(tmp, mask);
EIGEN_STRONG_INLINE Packet2f ptrunc<Packet2f>(const Packet2f& a) {
return vrnd_f32(a);
}
template <>
EIGEN_STRONG_INLINE Packet2f pfloor<Packet2f>(const Packet2f& a) {
const Packet2f cst_1 = pset1<Packet2f>(1.0f);
Packet2f tmp = print<Packet2f>(a);
// If greater, subtract one.
Packet2f mask = pcmp_lt(a, tmp);
mask = pand(mask, cst_1);
return psub(tmp, mask);
EIGEN_STRONG_INLINE Packet4f ptrunc<Packet4f>(const Packet4f& a) {
return vrndq_f32(a);
}
template <>
EIGEN_STRONG_INLINE Packet4f pceil<Packet4f>(const Packet4f& a) {
const Packet4f cst_1 = pset1<Packet4f>(1.0f);
Packet4f tmp = print<Packet4f>(a);
// If smaller, add one.
Packet4f mask = pcmp_lt(tmp, a);
mask = pand(mask, cst_1);
return padd(tmp, mask);
}
template <>
EIGEN_STRONG_INLINE Packet2f pceil<Packet2f>(const Packet2f& a) {
const Packet2f cst_1 = pset1<Packet2f>(1.0);
Packet2f tmp = print<Packet2f>(a);
// If smaller, add one.
Packet2f mask = pcmp_lt(tmp, a);
mask = pand(mask, cst_1);
return padd(tmp, mask);
}
#endif
/**
@@ -4800,10 +4744,6 @@ struct packet_traits<bfloat16> : default_packet_traits {
HasSetLinear = 1,
HasBlend = 0,
HasDiv = 1,
HasFloor = 1,
HasCeil = 1,
HasRint = 1,
HasSin = EIGEN_FAST_MATH,
HasCos = EIGEN_FAST_MATH,
HasLog = 1,
@@ -4983,6 +4923,16 @@ EIGEN_STRONG_INLINE Packet4bf pceil<Packet4bf>(const Packet4bf& a) {
return F32ToBf16(pceil<Packet4f>(Bf16ToF32(a)));
}
template <>
EIGEN_STRONG_INLINE Packet4bf pround<Packet4bf>(const Packet4bf& a) {
return F32ToBf16(pround<Packet4f>(Bf16ToF32(a)));
}
template <>
EIGEN_STRONG_INLINE Packet4bf ptrunc<Packet4bf>(const Packet4bf& a) {
return F32ToBf16(ptrunc<Packet4f>(Bf16ToF32(a)));
}
template <>
EIGEN_STRONG_INLINE Packet4bf pconj(const Packet4bf& a) {
return a;
@@ -5168,9 +5118,6 @@ struct packet_traits<double> : default_packet_traits {
HasBlend = 0,
HasDiv = 1,
HasFloor = 1,
HasCeil = 1,
HasRint = 1,
#if EIGEN_ARCH_ARM64 && !EIGEN_APPLE_DOUBLE_NEON_BUG
HasExp = 1,
@@ -5460,6 +5407,16 @@ EIGEN_STRONG_INLINE Packet2d pceil<Packet2d>(const Packet2d& a) {
return vrndpq_f64(a);
}
template <>
EIGEN_STRONG_INLINE Packet2d pround<Packet2d>(const Packet2d& a) {
return vrndaq_f64(a);
}
template <>
EIGEN_STRONG_INLINE Packet2d ptrunc<Packet2d>(const Packet2d& a) {
return vrndq_f64(a);
}
template <>
EIGEN_STRONG_INLINE Packet2d pldexp<Packet2d>(const Packet2d& a, const Packet2d& exponent) {
return pldexp_generic(a, exponent);
@@ -5521,9 +5478,6 @@ struct packet_traits<Eigen::half> : default_packet_traits {
HasInsert = 1,
HasReduxp = 1,
HasDiv = 1,
HasFloor = 1,
HasCeil = 1,
HasRint = 1,
HasSin = 0,
HasCos = 0,
HasLog = 0,
@@ -5791,6 +5745,26 @@ EIGEN_STRONG_INLINE Packet4hf pceil<Packet4hf>(const Packet4hf& a) {
return vrndp_f16(a);
}
template <>
EIGEN_STRONG_INLINE Packet8hf pround<Packet8hf>(const Packet8hf& a) {
return vrndaq_f16(a);
}
template <>
EIGEN_STRONG_INLINE Packet4hf pround<Packet4hf>(const Packet4hf& a) {
return vrnda_f16(a);
}
template <>
EIGEN_STRONG_INLINE Packet8hf ptrunc<Packet8hf>(const Packet8hf& a) {
return vrndq_f16(a);
}
template <>
EIGEN_STRONG_INLINE Packet4hf ptrunc<Packet4hf>(const Packet4hf& a) {
return vrnd_f16(a);
}
template <>
EIGEN_STRONG_INLINE Packet8hf psqrt<Packet8hf>(const Packet8hf& a) {
return vsqrtq_f16(a);

81
Eigen/src/Core/arch/SSE/PacketMath.h
View File

@@ -198,12 +198,6 @@ struct packet_traits<float> : default_packet_traits {
HasTanh = EIGEN_FAST_MATH,
HasErf = EIGEN_FAST_MATH,
HasBlend = 1,
HasCeil = 1,
HasFloor = 1,
#ifdef EIGEN_VECTORIZE_SSE4_1
HasRound = 1,
#endif
HasRint = 1,
HasSign = 0 // The manually vectorized version is slightly slower for SSE.
};
};
@@ -225,13 +219,7 @@ struct packet_traits<double> : default_packet_traits {
HasSqrt = 1,
HasRsqrt = 1,
HasATan = 1,
HasBlend = 1,
HasFloor = 1,
HasCeil = 1,
#ifdef EIGEN_VECTORIZE_SSE4_1
HasRound = 1,
#endif
HasRint = 1
HasBlend = 1
};
};
template <>
@@ -1309,73 +1297,14 @@ template <>
EIGEN_STRONG_INLINE Packet2d pfloor<Packet2d>(const Packet2d& a) {
return _mm_floor_pd(a);
}
#else
template <>
EIGEN_STRONG_INLINE Packet4f print(const Packet4f& a) {
// Adds and subtracts signum(a) * 2^23 to force rounding.
const Packet4f limit = pset1<Packet4f>(static_cast<float>(1 << 23));
const Packet4f abs_a = pabs(a);
Packet4f r = padd(abs_a, limit);
// Don't compile-away addition and subtraction.
EIGEN_OPTIMIZATION_BARRIER(r);
r = psub(r, limit);
// If greater than limit, simply return a. Otherwise, account for sign.
r = pselect(pcmp_lt(abs_a, limit), pselect(pcmp_lt(a, pzero(a)), pnegate(r), r), a);
return r;
}
template <>
EIGEN_STRONG_INLINE Packet2d print(const Packet2d& a) {
// Adds and subtracts signum(a) * 2^52 to force rounding.
const Packet2d limit = pset1<Packet2d>(static_cast<double>(1ull << 52));
const Packet2d abs_a = pabs(a);
Packet2d r = padd(abs_a, limit);
// Don't compile-away addition and subtraction.
EIGEN_OPTIMIZATION_BARRIER(r);
r = psub(r, limit);
// If greater than limit, simply return a. Otherwise, account for sign.
r = pselect(pcmp_lt(abs_a, limit), pselect(pcmp_lt(a, pzero(a)), pnegate(r), r), a);
return r;
EIGEN_STRONG_INLINE Packet4f ptrunc<Packet4f>(const Packet4f& a) {
return _mm_round_ps(a, _MM_FROUND_TRUNC);
}
template <>
EIGEN_STRONG_INLINE Packet4f pfloor<Packet4f>(const Packet4f& a) {
const Packet4f cst_1 = pset1<Packet4f>(1.0f);
Packet4f tmp = print<Packet4f>(a);
// If greater, subtract one.
Packet4f mask = _mm_cmpgt_ps(tmp, a);
mask = pand(mask, cst_1);
return psub(tmp, mask);
}
template <>
EIGEN_STRONG_INLINE Packet2d pfloor<Packet2d>(const Packet2d& a) {
const Packet2d cst_1 = pset1<Packet2d>(1.0);
Packet2d tmp = print<Packet2d>(a);
// If greater, subtract one.
Packet2d mask = _mm_cmpgt_pd(tmp, a);
mask = pand(mask, cst_1);
return psub(tmp, mask);
}
template <>
EIGEN_STRONG_INLINE Packet4f pceil<Packet4f>(const Packet4f& a) {
const Packet4f cst_1 = pset1<Packet4f>(1.0f);
Packet4f tmp = print<Packet4f>(a);
// If smaller, add one.
Packet4f mask = _mm_cmplt_ps(tmp, a);
mask = pand(mask, cst_1);
return padd(tmp, mask);
}
template <>
EIGEN_STRONG_INLINE Packet2d pceil<Packet2d>(const Packet2d& a) {
const Packet2d cst_1 = pset1<Packet2d>(1.0);
Packet2d tmp = print<Packet2d>(a);
// If smaller, add one.
Packet2d mask = _mm_cmplt_pd(tmp, a);
mask = pand(mask, cst_1);
return padd(tmp, mask);
EIGEN_STRONG_INLINE Packet2d ptrunc<Packet2d>(const Packet2d& a) {
return _mm_round_pd(a, _MM_FROUND_TRUNC);
}
#endif

1
Eigen/src/Core/arch/SVE/PacketMath.h
View File

@@ -353,7 +353,6 @@ struct packet_traits<float> : default_packet_traits {
HasReduxp = 0, // Not implemented in SVE
HasDiv = 1,
HasFloor = 1,
HasSin = EIGEN_FAST_MATH,
HasCos = EIGEN_FAST_MATH,

6
Eigen/src/Core/arch/ZVector/PacketMath.h
View File

@@ -195,9 +195,6 @@ struct packet_traits<float> : default_packet_traits {
HasRsqrt = 1,
HasTanh = 1,
HasErf = 1,
HasRound = 1,
HasFloor = 1,
HasCeil = 1,
HasNegate = 1,
HasBlend = 1
};
@@ -225,9 +222,6 @@ struct packet_traits<double> : default_packet_traits {
HasExp = 1,
HasSqrt = 1,
HasRsqrt = 1,
HasRound = 1,
HasFloor = 1,
HasCeil = 1,
HasNegate = 1,
HasBlend = 1
};