mirror of
https://gitlab.com/libeigen/eigen.git
synced 2026-04-10 11:34:33 +08:00
a566074480
This change re-instates the fast rational approximation of the logistic function for float32 in Eigen (removed in 66f07efeae), but uses the more accurate approximation 1/(1+exp(-1)) ~= exp(x) below -9. The exponential is only calculated on the vectorized path if at least one element in the SIMD input vector is less than -9.
This change also contains a few improvements to speed up the original float specialization of logistic:
- Introduce EIGEN_PREDICT_{FALSE,TRUE} for __builtin_predict and use it to predict that the logistic-only path is most likely (~2-3% speedup for the common case).
- Carefully set the upper clipping point to the smallest x where the approximation evaluates to exactly 1. This saves the explicit clamping of the output (~7% speedup).
The increased accuracy for tanh comes at a cost of 10-20% depending on instruction set.
The benchmarks below repeated calls
u = v.logistic() (u = v.tanh(), respectively)
where u and v are of type Eigen::ArrayXf, have length 8k, and v contains random numbers in [-1,1].
Benchmark numbers for logistic:
Before:
Benchmark Time(ns) CPU(ns) Iterations
-----------------------------------------------------------------
SSE
BM_eigen_logistic_float 4467 4468 155835 model_time: 4827
AVX
BM_eigen_logistic_float 2347 2347 299135 model_time: 2926
AVX+FMA
BM_eigen_logistic_float 1467 1467 476143 model_time: 2926
AVX512
BM_eigen_logistic_float 805 805 858696 model_time: 1463
After:
Benchmark Time(ns) CPU(ns) Iterations
-----------------------------------------------------------------
SSE
BM_eigen_logistic_float 2589 2590 270264 model_time: 4827
AVX
BM_eigen_logistic_float 1428 1428 489265 model_time: 2926
AVX+FMA
BM_eigen_logistic_float 1059 1059 662255 model_time: 2926
AVX512
BM_eigen_logistic_float 673 673 1000000 model_time: 1463
Benchmark numbers for tanh:
Before:
Benchmark Time(ns) CPU(ns) Iterations
-----------------------------------------------------------------
SSE
BM_eigen_tanh_float 2391 2391 292624 model_time: 4242
AVX
BM_eigen_tanh_float 1256 1256 554662 model_time: 2633
AVX+FMA
BM_eigen_tanh_float 823 823 866267 model_time: 1609
AVX512
BM_eigen_tanh_float 443 443 1578999 model_time: 805
After:
Benchmark Time(ns) CPU(ns) Iterations
-----------------------------------------------------------------
SSE
BM_eigen_tanh_float 2588 2588 273531 model_time: 4242
AVX
BM_eigen_tanh_float 1536 1536 452321 model_time: 2633
AVX+FMA
BM_eigen_tanh_float 1007 1007 694681 model_time: 1609
AVX512
BM_eigen_tanh_float 471 471 1472178 model_time: 805
107 lines
3.5 KiB
C++
107 lines
3.5 KiB
C++
// This file is part of Eigen, a lightweight C++ template library
|
|
// for linear algebra.
|
|
//
|
|
// Copyright (C) 2014 Pedro Gonnet (pedro.gonnet@gmail.com)
|
|
// Copyright (C) 2016 Gael Guennebaud <gael.guennebaud@inria.fr>
|
|
//
|
|
// This Source Code Form is subject to the terms of the Mozilla
|
|
// Public License v. 2.0. If a copy of the MPL was not distributed
|
|
// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
|
|
|
#ifndef EIGEN_MATHFUNCTIONSIMPL_H
|
|
#define EIGEN_MATHFUNCTIONSIMPL_H
|
|
|
|
namespace Eigen {
|
|
|
|
namespace internal {
|
|
|
|
/** \internal \returns the hyperbolic tan of \a a (coeff-wise)
|
|
Doesn't do anything fancy, just a 13/6-degree rational interpolant which
|
|
is accurate up to a couple of ulps in the (approximate) range [-8, 8],
|
|
outside of which tanh(x) = +/-1 in single precision. The input is clamped
|
|
to the range [-c, c]. The value c is chosen as the smallest value where
|
|
the approximation evaluates to exactly 1. In the reange [-0.0004, 0.0004]
|
|
the approxmation tanh(x) ~= x is used for better accuracy as x tends to zero.
|
|
|
|
This implementation works on both scalars and packets.
|
|
*/
|
|
template<typename T>
|
|
T generic_fast_tanh_float(const T& a_x)
|
|
{
|
|
// Clamp the inputs to the range [-c, c]
|
|
#ifdef EIGEN_VECTORIZE_FMA
|
|
const T plus_clamp = pset1<T>(7.99881172180175781f);
|
|
const T minus_clamp = pset1<T>(-7.99881172180175781f);
|
|
#else
|
|
const T plus_clamp = pset1<T>(7.90531110763549805f);
|
|
const T minus_clamp = pset1<T>(-7.90531110763549805f);
|
|
#endif
|
|
const T tiny = pset1<T>(0.0004f);
|
|
const T x = pmax(pmin(a_x, plus_clamp), minus_clamp);
|
|
const T tiny_mask = pcmp_lt(pabs(a_x), tiny);
|
|
// The monomial coefficients of the numerator polynomial (odd).
|
|
const T alpha_1 = pset1<T>(4.89352455891786e-03f);
|
|
const T alpha_3 = pset1<T>(6.37261928875436e-04f);
|
|
const T alpha_5 = pset1<T>(1.48572235717979e-05f);
|
|
const T alpha_7 = pset1<T>(5.12229709037114e-08f);
|
|
const T alpha_9 = pset1<T>(-8.60467152213735e-11f);
|
|
const T alpha_11 = pset1<T>(2.00018790482477e-13f);
|
|
const T alpha_13 = pset1<T>(-2.76076847742355e-16f);
|
|
|
|
// The monomial coefficients of the denominator polynomial (even).
|
|
const T beta_0 = pset1<T>(4.89352518554385e-03f);
|
|
const T beta_2 = pset1<T>(2.26843463243900e-03f);
|
|
const T beta_4 = pset1<T>(1.18534705686654e-04f);
|
|
const T beta_6 = pset1<T>(1.19825839466702e-06f);
|
|
|
|
// Since the polynomials are odd/even, we need x^2.
|
|
const T x2 = pmul(x, x);
|
|
|
|
// Evaluate the numerator polynomial p.
|
|
T p = pmadd(x2, alpha_13, alpha_11);
|
|
p = pmadd(x2, p, alpha_9);
|
|
p = pmadd(x2, p, alpha_7);
|
|
p = pmadd(x2, p, alpha_5);
|
|
p = pmadd(x2, p, alpha_3);
|
|
p = pmadd(x2, p, alpha_1);
|
|
p = pmul(x, p);
|
|
|
|
// Evaluate the denominator polynomial q.
|
|
T q = pmadd(x2, beta_6, beta_4);
|
|
q = pmadd(x2, q, beta_2);
|
|
q = pmadd(x2, q, beta_0);
|
|
|
|
// Divide the numerator by the denominator.
|
|
return pselect(tiny_mask, x, pdiv(p, q));
|
|
}
|
|
|
|
template<typename RealScalar>
|
|
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
|
|
RealScalar positive_real_hypot(const RealScalar& x, const RealScalar& y)
|
|
{
|
|
EIGEN_USING_STD_MATH(sqrt);
|
|
RealScalar p, qp;
|
|
p = numext::maxi(x,y);
|
|
if(p==RealScalar(0)) return RealScalar(0);
|
|
qp = numext::mini(y,x) / p;
|
|
return p * sqrt(RealScalar(1) + qp*qp);
|
|
}
|
|
|
|
template<typename Scalar>
|
|
struct hypot_impl
|
|
{
|
|
typedef typename NumTraits<Scalar>::Real RealScalar;
|
|
static EIGEN_DEVICE_FUNC
|
|
inline RealScalar run(const Scalar& x, const Scalar& y)
|
|
{
|
|
EIGEN_USING_STD_MATH(abs);
|
|
return positive_real_hypot<RealScalar>(abs(x), abs(y));
|
|
}
|
|
};
|
|
|
|
} // end namespace internal
|
|
|
|
} // end namespace Eigen
|
|
|
|
#endif // EIGEN_MATHFUNCTIONSIMPL_H
|