mirror of
https://gitlab.com/libeigen/eigen.git
synced 2026-04-10 11:34:33 +08:00
Merged default into unary-array-cwise-functors
This commit is contained in:
Deanna Hood
@@ -647,11 +647,15 @@ struct evaluator<Map<PlainObjectType, MapOptions, StrideType> >
|
||||
HasNoStride = HasNoInnerStride && HasNoOuterStride,
|
||||
IsAligned = bool(EIGEN_ALIGN) && ((int(MapOptions)&Aligned)==Aligned),
|
||||
IsDynamicSize = PlainObjectType::SizeAtCompileTime==Dynamic,
|
||||
|
||||
// TODO: should check for smaller packet types once we can handle multi-sized packet types
|
||||
AlignBytes = int(packet_traits<Scalar>::size) * sizeof(Scalar),
|
||||
|
||||
KeepsPacketAccess = bool(HasNoInnerStride)
|
||||
&& ( bool(IsDynamicSize)
|
||||
|| HasNoOuterStride
|
||||
|| ( OuterStrideAtCompileTime!=Dynamic
|
||||
&& ((static_cast<int>(sizeof(Scalar))*OuterStrideAtCompileTime)%EIGEN_ALIGN_BYTES)==0 ) ),
|
||||
&& ((static_cast<int>(sizeof(Scalar))*OuterStrideAtCompileTime) % AlignBytes)==0 ) ),
|
||||
Flags0 = evaluator<PlainObjectType>::Flags,
|
||||
Flags1 = IsAligned ? (int(Flags0) | AlignedBit) : (int(Flags0) & ~AlignedBit),
|
||||
Flags2 = (bool(HasNoStride) || bool(PlainObjectType::IsVectorAtCompileTime))
|
||||
@@ -717,7 +721,10 @@ struct evaluator<Block<ArgType, BlockRows, BlockCols, InnerPanel> >
|
||||
&& (InnerStrideAtCompileTime == 1)
|
||||
? PacketAccessBit : 0,
|
||||
|
||||
MaskAlignedBit = (InnerPanel && (OuterStrideAtCompileTime!=Dynamic) && (((OuterStrideAtCompileTime * int(sizeof(Scalar))) % EIGEN_ALIGN_BYTES) == 0)) ? AlignedBit : 0,
|
||||
// TODO: should check for smaller packet types once we can handle multi-sized packet types
|
||||
AlignBytes = int(packet_traits<Scalar>::size) * sizeof(Scalar),
|
||||
|
||||
MaskAlignedBit = (InnerPanel && (OuterStrideAtCompileTime!=Dynamic) && (((OuterStrideAtCompileTime * int(sizeof(Scalar))) % AlignBytes) == 0)) ? AlignedBit : 0,
|
||||
FlagsLinearAccessBit = (RowsAtCompileTime == 1 || ColsAtCompileTime == 1 || (InnerPanel && (evaluator<ArgType>::Flags&LinearAccessBit))) ? LinearAccessBit : 0,
|
||||
FlagsRowMajorBit = XprType::Flags&RowMajorBit,
|
||||
Flags0 = evaluator<ArgType>::Flags & ( (HereditaryBits & ~RowMajorBit) |
|
||||
@@ -825,12 +832,16 @@ struct block_evaluator<ArgType, BlockRows, BlockCols, InnerPanel, /* HasDirectAc
|
||||
typename Block<ArgType, BlockRows, BlockCols, InnerPanel>::PlainObject>
|
||||
{
|
||||
typedef Block<ArgType, BlockRows, BlockCols, InnerPanel> XprType;
|
||||
typedef typename XprType::Scalar Scalar;
|
||||
|
||||
EIGEN_DEVICE_FUNC explicit block_evaluator(const XprType& block)
|
||||
: mapbase_evaluator<XprType, typename XprType::PlainObject>(block)
|
||||
{
|
||||
// TODO: should check for smaller packet types once we can handle multi-sized packet types
|
||||
const int AlignBytes = int(packet_traits<Scalar>::size) * sizeof(Scalar);
|
||||
EIGEN_ONLY_USED_FOR_DEBUG(AlignBytes)
|
||||
// FIXME this should be an internal assertion
|
||||
eigen_assert(EIGEN_IMPLIES(evaluator<XprType>::Flags&AlignedBit, (size_t(block.data()) % EIGEN_ALIGN_BYTES) == 0) && "data is not aligned");
|
||||
eigen_assert(EIGEN_IMPLIES(evaluator<XprType>::Flags&AlignedBit, (size_t(block.data()) % AlignBytes) == 0) && "data is not aligned");
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
@@ -300,9 +300,10 @@ template<typename Derived>
|
||||
bool DenseBase<Derived>::isApproxToConstant
|
||||
(const Scalar& val, const RealScalar& prec) const
|
||||
{
|
||||
typename internal::nested_eval<Derived,1>::type self(derived());
|
||||
for(Index j = 0; j < cols(); ++j)
|
||||
for(Index i = 0; i < rows(); ++i)
|
||||
if(!internal::isApprox(this->coeff(i, j), val, prec))
|
||||
if(!internal::isApprox(self.coeff(i, j), val, prec))
|
||||
return false;
|
||||
return true;
|
||||
}
|
||||
@@ -484,9 +485,10 @@ DenseBase<Derived>::Zero()
|
||||
template<typename Derived>
|
||||
bool DenseBase<Derived>::isZero(const RealScalar& prec) const
|
||||
{
|
||||
typename internal::nested_eval<Derived,1>::type self(derived());
|
||||
for(Index j = 0; j < cols(); ++j)
|
||||
for(Index i = 0; i < rows(); ++i)
|
||||
if(!internal::isMuchSmallerThan(this->coeff(i, j), static_cast<Scalar>(1), prec))
|
||||
if(!internal::isMuchSmallerThan(self.coeff(i, j), static_cast<Scalar>(1), prec))
|
||||
return false;
|
||||
return true;
|
||||
}
|
||||
@@ -719,18 +721,19 @@ template<typename Derived>
|
||||
bool MatrixBase<Derived>::isIdentity
|
||||
(const RealScalar& prec) const
|
||||
{
|
||||
typename internal::nested_eval<Derived,1>::type self(derived());
|
||||
for(Index j = 0; j < cols(); ++j)
|
||||
{
|
||||
for(Index i = 0; i < rows(); ++i)
|
||||
{
|
||||
if(i == j)
|
||||
{
|
||||
if(!internal::isApprox(this->coeff(i, j), static_cast<Scalar>(1), prec))
|
||||
if(!internal::isApprox(self.coeff(i, j), static_cast<Scalar>(1), prec))
|
||||
return false;
|
||||
}
|
||||
else
|
||||
{
|
||||
if(!internal::isMuchSmallerThan(this->coeff(i, j), static_cast<RealScalar>(1), prec))
|
||||
if(!internal::isMuchSmallerThan(self.coeff(i, j), static_cast<RealScalar>(1), prec))
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -34,14 +34,35 @@ void check_static_allocation_size()
|
||||
#endif
|
||||
}
|
||||
|
||||
template<typename T, int Size, typename Packet = typename packet_traits<T>::type,
|
||||
bool Match = bool((Size%unpacket_traits<Packet>::size)==0),
|
||||
bool TryHalf = bool(int(unpacket_traits<Packet>::size) > 1)
|
||||
&& bool(int(unpacket_traits<Packet>::size) > int(unpacket_traits<typename unpacket_traits<Packet>::half>::size)) >
|
||||
struct compute_default_alignment
|
||||
{
|
||||
enum { value = 0 };
|
||||
};
|
||||
|
||||
template<typename T, int Size, typename Packet, bool TryHalf>
|
||||
struct compute_default_alignment<T, Size, Packet, true, TryHalf> // Match
|
||||
{
|
||||
enum { value = sizeof(T) * unpacket_traits<Packet>::size };
|
||||
};
|
||||
|
||||
template<typename T, int Size, typename Packet>
|
||||
struct compute_default_alignment<T, Size, Packet, false, true> // Try-half
|
||||
{
|
||||
// current packet too large, try with an half-packet
|
||||
enum { value = compute_default_alignment<T, Size, typename unpacket_traits<Packet>::half>::value };
|
||||
};
|
||||
|
||||
/** \internal
|
||||
* Static array. If the MatrixOrArrayOptions require auto-alignment, the array will be automatically aligned:
|
||||
* to 16 bytes boundary if the total size is a multiple of 16 bytes.
|
||||
*/
|
||||
template <typename T, int Size, int MatrixOrArrayOptions,
|
||||
int Alignment = (MatrixOrArrayOptions&DontAlign) ? 0
|
||||
: (((Size*sizeof(T))%EIGEN_ALIGN_BYTES)==0) ? EIGEN_ALIGN_BYTES
|
||||
: 0 >
|
||||
: compute_default_alignment<T,Size>::value >
|
||||
struct plain_array
|
||||
{
|
||||
T array[Size];
|
||||
@@ -81,14 +102,71 @@ struct plain_array
|
||||
#endif
|
||||
|
||||
template <typename T, int Size, int MatrixOrArrayOptions>
|
||||
struct plain_array<T, Size, MatrixOrArrayOptions, EIGEN_ALIGN_BYTES>
|
||||
struct plain_array<T, Size, MatrixOrArrayOptions, 8>
|
||||
{
|
||||
EIGEN_USER_ALIGN_DEFAULT T array[Size];
|
||||
EIGEN_ALIGN_TO_BOUNDARY(8) T array[Size];
|
||||
|
||||
EIGEN_DEVICE_FUNC
|
||||
plain_array()
|
||||
{
|
||||
EIGEN_MAKE_UNALIGNED_ARRAY_ASSERT(EIGEN_ALIGN_BYTES-1);
|
||||
EIGEN_MAKE_UNALIGNED_ARRAY_ASSERT(7);
|
||||
check_static_allocation_size<T,Size>();
|
||||
}
|
||||
|
||||
EIGEN_DEVICE_FUNC
|
||||
plain_array(constructor_without_unaligned_array_assert)
|
||||
{
|
||||
check_static_allocation_size<T,Size>();
|
||||
}
|
||||
};
|
||||
|
||||
template <typename T, int Size, int MatrixOrArrayOptions>
|
||||
struct plain_array<T, Size, MatrixOrArrayOptions, 16>
|
||||
{
|
||||
EIGEN_ALIGN_TO_BOUNDARY(16) T array[Size];
|
||||
|
||||
EIGEN_DEVICE_FUNC
|
||||
plain_array()
|
||||
{
|
||||
EIGEN_MAKE_UNALIGNED_ARRAY_ASSERT(15);
|
||||
check_static_allocation_size<T,Size>();
|
||||
}
|
||||
|
||||
EIGEN_DEVICE_FUNC
|
||||
plain_array(constructor_without_unaligned_array_assert)
|
||||
{
|
||||
check_static_allocation_size<T,Size>();
|
||||
}
|
||||
};
|
||||
|
||||
template <typename T, int Size, int MatrixOrArrayOptions>
|
||||
struct plain_array<T, Size, MatrixOrArrayOptions, 32>
|
||||
{
|
||||
EIGEN_ALIGN_TO_BOUNDARY(32) T array[Size];
|
||||
|
||||
EIGEN_DEVICE_FUNC
|
||||
plain_array()
|
||||
{
|
||||
EIGEN_MAKE_UNALIGNED_ARRAY_ASSERT(31);
|
||||
check_static_allocation_size<T,Size>();
|
||||
}
|
||||
|
||||
EIGEN_DEVICE_FUNC
|
||||
plain_array(constructor_without_unaligned_array_assert)
|
||||
{
|
||||
check_static_allocation_size<T,Size>();
|
||||
}
|
||||
};
|
||||
|
||||
template <typename T, int Size, int MatrixOrArrayOptions>
|
||||
struct plain_array<T, Size, MatrixOrArrayOptions, 64>
|
||||
{
|
||||
EIGEN_ALIGN_TO_BOUNDARY(64) T array[Size];
|
||||
|
||||
EIGEN_DEVICE_FUNC
|
||||
plain_array()
|
||||
{
|
||||
EIGEN_MAKE_UNALIGNED_ARRAY_ASSERT(63);
|
||||
check_static_allocation_size<T,Size>();
|
||||
}
|
||||
|
||||
|
||||
@@ -224,13 +224,13 @@ bool MatrixBase<Derived>::isOrthogonal
|
||||
template<typename Derived>
|
||||
bool MatrixBase<Derived>::isUnitary(const RealScalar& prec) const
|
||||
{
|
||||
typename Derived::Nested nested(derived());
|
||||
typename internal::nested_eval<Derived,1>::type self(derived());
|
||||
for(Index i = 0; i < cols(); ++i)
|
||||
{
|
||||
if(!internal::isApprox(nested.col(i).squaredNorm(), static_cast<RealScalar>(1), prec))
|
||||
if(!internal::isApprox(self.col(i).squaredNorm(), static_cast<RealScalar>(1), prec))
|
||||
return false;
|
||||
for(Index j = 0; j < i; ++j)
|
||||
if(!internal::isMuchSmallerThan(nested.col(i).dot(nested.col(j)), static_cast<Scalar>(1), prec))
|
||||
if(!internal::isMuchSmallerThan(self.col(i).dot(self.col(j)), static_cast<Scalar>(1), prec))
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
|
||||
@@ -328,6 +328,7 @@ struct hypot_impl
|
||||
p = _y;
|
||||
qp = _x / p;
|
||||
}
|
||||
if(p==RealScalar(0)) return RealScalar(0);
|
||||
return p * sqrt(RealScalar(1) + qp*qp);
|
||||
}
|
||||
};
|
||||
@@ -560,48 +561,48 @@ struct random_default_impl<Scalar, false, false>
|
||||
};
|
||||
|
||||
enum {
|
||||
floor_log2_terminate,
|
||||
floor_log2_move_up,
|
||||
floor_log2_move_down,
|
||||
floor_log2_bogus
|
||||
meta_floor_log2_terminate,
|
||||
meta_floor_log2_move_up,
|
||||
meta_floor_log2_move_down,
|
||||
meta_floor_log2_bogus
|
||||
};
|
||||
|
||||
template<unsigned int n, int lower, int upper> struct floor_log2_selector
|
||||
template<unsigned int n, int lower, int upper> struct meta_floor_log2_selector
|
||||
{
|
||||
enum { middle = (lower + upper) / 2,
|
||||
value = (upper <= lower + 1) ? int(floor_log2_terminate)
|
||||
: (n < (1 << middle)) ? int(floor_log2_move_down)
|
||||
: (n==0) ? int(floor_log2_bogus)
|
||||
: int(floor_log2_move_up)
|
||||
value = (upper <= lower + 1) ? int(meta_floor_log2_terminate)
|
||||
: (n < (1 << middle)) ? int(meta_floor_log2_move_down)
|
||||
: (n==0) ? int(meta_floor_log2_bogus)
|
||||
: int(meta_floor_log2_move_up)
|
||||
};
|
||||
};
|
||||
|
||||
template<unsigned int n,
|
||||
int lower = 0,
|
||||
int upper = sizeof(unsigned int) * CHAR_BIT - 1,
|
||||
int selector = floor_log2_selector<n, lower, upper>::value>
|
||||
struct floor_log2 {};
|
||||
int selector = meta_floor_log2_selector<n, lower, upper>::value>
|
||||
struct meta_floor_log2 {};
|
||||
|
||||
template<unsigned int n, int lower, int upper>
|
||||
struct floor_log2<n, lower, upper, floor_log2_move_down>
|
||||
struct meta_floor_log2<n, lower, upper, meta_floor_log2_move_down>
|
||||
{
|
||||
enum { value = floor_log2<n, lower, floor_log2_selector<n, lower, upper>::middle>::value };
|
||||
enum { value = meta_floor_log2<n, lower, meta_floor_log2_selector<n, lower, upper>::middle>::value };
|
||||
};
|
||||
|
||||
template<unsigned int n, int lower, int upper>
|
||||
struct floor_log2<n, lower, upper, floor_log2_move_up>
|
||||
struct meta_floor_log2<n, lower, upper, meta_floor_log2_move_up>
|
||||
{
|
||||
enum { value = floor_log2<n, floor_log2_selector<n, lower, upper>::middle, upper>::value };
|
||||
enum { value = meta_floor_log2<n, meta_floor_log2_selector<n, lower, upper>::middle, upper>::value };
|
||||
};
|
||||
|
||||
template<unsigned int n, int lower, int upper>
|
||||
struct floor_log2<n, lower, upper, floor_log2_terminate>
|
||||
struct meta_floor_log2<n, lower, upper, meta_floor_log2_terminate>
|
||||
{
|
||||
enum { value = (n >= ((unsigned int)(1) << (lower+1))) ? lower+1 : lower };
|
||||
};
|
||||
|
||||
template<unsigned int n, int lower, int upper>
|
||||
struct floor_log2<n, lower, upper, floor_log2_bogus>
|
||||
struct meta_floor_log2<n, lower, upper, meta_floor_log2_bogus>
|
||||
{
|
||||
// no value, error at compile time
|
||||
};
|
||||
@@ -609,11 +610,24 @@ struct floor_log2<n, lower, upper, floor_log2_bogus>
|
||||
template<typename Scalar>
|
||||
struct random_default_impl<Scalar, false, true>
|
||||
{
|
||||
typedef typename NumTraits<Scalar>::NonInteger NonInteger;
|
||||
|
||||
static inline Scalar run(const Scalar& x, const Scalar& y)
|
||||
{
|
||||
return x + Scalar((NonInteger(y)-x+1) * std::rand() / (RAND_MAX + NonInteger(1)));
|
||||
{
|
||||
using std::max;
|
||||
using std::min;
|
||||
typedef typename conditional<NumTraits<Scalar>::IsSigned,std::ptrdiff_t,std::size_t>::type ScalarX;
|
||||
if(y<x)
|
||||
return x;
|
||||
std::size_t range = ScalarX(y)-ScalarX(x);
|
||||
std::size_t offset = 0;
|
||||
// rejection sampling
|
||||
std::size_t divisor = (range+RAND_MAX-1)/(range+1);
|
||||
std::size_t multiplier = (range+RAND_MAX-1)/std::size_t(RAND_MAX);
|
||||
|
||||
do {
|
||||
offset = ( (std::size_t(std::rand()) * multiplier) / divisor );
|
||||
} while (offset > range);
|
||||
|
||||
return Scalar(ScalarX(x) + offset);
|
||||
}
|
||||
|
||||
static inline Scalar run()
|
||||
@@ -621,7 +635,7 @@ struct random_default_impl<Scalar, false, true>
|
||||
#ifdef EIGEN_MAKING_DOCS
|
||||
return run(Scalar(NumTraits<Scalar>::IsSigned ? -10 : 0), Scalar(10));
|
||||
#else
|
||||
enum { rand_bits = floor_log2<(unsigned int)(RAND_MAX)+1>::value,
|
||||
enum { rand_bits = meta_floor_log2<(unsigned int)(RAND_MAX)+1>::value,
|
||||
scalar_bits = sizeof(Scalar) * CHAR_BIT,
|
||||
shift = EIGEN_PLAIN_ENUM_MAX(0, int(rand_bits) - int(scalar_bits)),
|
||||
offset = NumTraits<Scalar>::IsSigned ? (1 << (EIGEN_PLAIN_ENUM_MIN(rand_bits,scalar_bits)-1)) : 0
|
||||
|
||||
@@ -409,7 +409,8 @@ struct product_evaluator<Product<Lhs, Rhs, LazyProduct>, ProductTag, DenseShape,
|
||||
|
||||
LhsCoeffReadCost = LhsEtorType::CoeffReadCost,
|
||||
RhsCoeffReadCost = RhsEtorType::CoeffReadCost,
|
||||
CoeffReadCost = (InnerSize == Dynamic || LhsCoeffReadCost==Dynamic || RhsCoeffReadCost==Dynamic || NumTraits<Scalar>::AddCost==Dynamic || NumTraits<Scalar>::MulCost==Dynamic) ? Dynamic
|
||||
CoeffReadCost = InnerSize==0 ? NumTraits<Scalar>::ReadCost
|
||||
: (InnerSize == Dynamic || LhsCoeffReadCost==Dynamic || RhsCoeffReadCost==Dynamic || NumTraits<Scalar>::AddCost==Dynamic || NumTraits<Scalar>::MulCost==Dynamic) ? Dynamic
|
||||
: InnerSize * (NumTraits<Scalar>::MulCost + LhsCoeffReadCost + RhsCoeffReadCost)
|
||||
+ (InnerSize - 1) * NumTraits<Scalar>::AddCost,
|
||||
|
||||
@@ -484,7 +485,7 @@ struct product_evaluator<Product<Lhs, Rhs, LazyProduct>, ProductTag, DenseShape,
|
||||
{
|
||||
PacketScalar res;
|
||||
typedef etor_product_packet_impl<Flags&RowMajorBit ? RowMajor : ColMajor,
|
||||
Unroll ? InnerSize-1 : Dynamic,
|
||||
Unroll ? InnerSize : Dynamic,
|
||||
LhsEtorType, RhsEtorType, PacketScalar, LoadMode> PacketImpl;
|
||||
|
||||
PacketImpl::run(row, col, m_lhsImpl, m_rhsImpl, m_innerDim, res);
|
||||
@@ -527,7 +528,7 @@ struct etor_product_packet_impl<RowMajor, UnrollingIndex, Lhs, Rhs, Packet, Load
|
||||
static EIGEN_STRONG_INLINE void run(Index row, Index col, const Lhs& lhs, const Rhs& rhs, Index innerDim, Packet &res)
|
||||
{
|
||||
etor_product_packet_impl<RowMajor, UnrollingIndex-1, Lhs, Rhs, Packet, LoadMode>::run(row, col, lhs, rhs, innerDim, res);
|
||||
res = pmadd(pset1<Packet>(lhs.coeff(row, UnrollingIndex)), rhs.template packet<LoadMode>(UnrollingIndex, col), res);
|
||||
res = pmadd(pset1<Packet>(lhs.coeff(row, UnrollingIndex-1)), rhs.template packet<LoadMode>(UnrollingIndex-1, col), res);
|
||||
}
|
||||
};
|
||||
|
||||
@@ -537,12 +538,12 @@ struct etor_product_packet_impl<ColMajor, UnrollingIndex, Lhs, Rhs, Packet, Load
|
||||
static EIGEN_STRONG_INLINE void run(Index row, Index col, const Lhs& lhs, const Rhs& rhs, Index innerDim, Packet &res)
|
||||
{
|
||||
etor_product_packet_impl<ColMajor, UnrollingIndex-1, Lhs, Rhs, Packet, LoadMode>::run(row, col, lhs, rhs, innerDim, res);
|
||||
res = pmadd(lhs.template packet<LoadMode>(row, UnrollingIndex), pset1<Packet>(rhs.coeff(UnrollingIndex, col)), res);
|
||||
res = pmadd(lhs.template packet<LoadMode>(row, UnrollingIndex-1), pset1<Packet>(rhs.coeff(UnrollingIndex-1, col)), res);
|
||||
}
|
||||
};
|
||||
|
||||
template<typename Lhs, typename Rhs, typename Packet, int LoadMode>
|
||||
struct etor_product_packet_impl<RowMajor, 0, Lhs, Rhs, Packet, LoadMode>
|
||||
struct etor_product_packet_impl<RowMajor, 1, Lhs, Rhs, Packet, LoadMode>
|
||||
{
|
||||
static EIGEN_STRONG_INLINE void run(Index row, Index col, const Lhs& lhs, const Rhs& rhs, Index /*innerDim*/, Packet &res)
|
||||
{
|
||||
@@ -551,7 +552,7 @@ struct etor_product_packet_impl<RowMajor, 0, Lhs, Rhs, Packet, LoadMode>
|
||||
};
|
||||
|
||||
template<typename Lhs, typename Rhs, typename Packet, int LoadMode>
|
||||
struct etor_product_packet_impl<ColMajor, 0, Lhs, Rhs, Packet, LoadMode>
|
||||
struct etor_product_packet_impl<ColMajor, 1, Lhs, Rhs, Packet, LoadMode>
|
||||
{
|
||||
static EIGEN_STRONG_INLINE void run(Index row, Index col, const Lhs& lhs, const Rhs& rhs, Index /*innerDim*/, Packet &res)
|
||||
{
|
||||
@@ -559,14 +560,31 @@ struct etor_product_packet_impl<ColMajor, 0, Lhs, Rhs, Packet, LoadMode>
|
||||
}
|
||||
};
|
||||
|
||||
template<typename Lhs, typename Rhs, typename Packet, int LoadMode>
|
||||
struct etor_product_packet_impl<RowMajor, 0, Lhs, Rhs, Packet, LoadMode>
|
||||
{
|
||||
static EIGEN_STRONG_INLINE void run(Index /*row*/, Index /*col*/, const Lhs& /*lhs*/, const Rhs& /*rhs*/, Index /*innerDim*/, Packet &res)
|
||||
{
|
||||
res = pset1<Packet>(0);
|
||||
}
|
||||
};
|
||||
|
||||
template<typename Lhs, typename Rhs, typename Packet, int LoadMode>
|
||||
struct etor_product_packet_impl<ColMajor, 0, Lhs, Rhs, Packet, LoadMode>
|
||||
{
|
||||
static EIGEN_STRONG_INLINE void run(Index /*row*/, Index /*col*/, const Lhs& /*lhs*/, const Rhs& /*rhs*/, Index /*innerDim*/, Packet &res)
|
||||
{
|
||||
res = pset1<Packet>(0);
|
||||
}
|
||||
};
|
||||
|
||||
template<typename Lhs, typename Rhs, typename Packet, int LoadMode>
|
||||
struct etor_product_packet_impl<RowMajor, Dynamic, Lhs, Rhs, Packet, LoadMode>
|
||||
{
|
||||
static EIGEN_STRONG_INLINE void run(Index row, Index col, const Lhs& lhs, const Rhs& rhs, Index innerDim, Packet& res)
|
||||
{
|
||||
eigen_assert(innerDim>0 && "you are using a non initialized matrix");
|
||||
res = pmul(pset1<Packet>(lhs.coeff(row, 0)),rhs.template packet<LoadMode>(0, col));
|
||||
for(Index i = 1; i < innerDim; ++i)
|
||||
res = pset1<Packet>(0);
|
||||
for(Index i = 0; i < innerDim; ++i)
|
||||
res = pmadd(pset1<Packet>(lhs.coeff(row, i)), rhs.template packet<LoadMode>(i, col), res);
|
||||
}
|
||||
};
|
||||
@@ -576,9 +594,8 @@ struct etor_product_packet_impl<ColMajor, Dynamic, Lhs, Rhs, Packet, LoadMode>
|
||||
{
|
||||
static EIGEN_STRONG_INLINE void run(Index row, Index col, const Lhs& lhs, const Rhs& rhs, Index innerDim, Packet& res)
|
||||
{
|
||||
eigen_assert(innerDim>0 && "you are using a non initialized matrix");
|
||||
res = pmul(lhs.template packet<LoadMode>(row, 0), pset1<Packet>(rhs.coeff(0, col)));
|
||||
for(Index i = 1; i < innerDim; ++i)
|
||||
res = pset1<Packet>(0);
|
||||
for(Index i = 0; i < innerDim; ++i)
|
||||
res = pmadd(lhs.template packet<LoadMode>(row, i), pset1<Packet>(rhs.coeff(i, col)), res);
|
||||
}
|
||||
};
|
||||
@@ -678,8 +695,7 @@ public:
|
||||
//_Vectorizable = bool(int(MatrixFlags)&PacketAccessBit) && ((!_PacketOnDiag) || (_SameTypes && bool(int(DiagFlags)&PacketAccessBit))),
|
||||
_Vectorizable = bool(int(MatrixFlags)&PacketAccessBit) && _SameTypes && (_ScalarAccessOnDiag || (bool(int(DiagFlags)&PacketAccessBit))),
|
||||
_LinearAccessMask = (MatrixType::RowsAtCompileTime==1 || MatrixType::ColsAtCompileTime==1) ? LinearAccessBit : 0,
|
||||
Flags = ((HereditaryBits|_LinearAccessMask) & (unsigned int)(MatrixFlags)) | (_Vectorizable ? PacketAccessBit : 0) | AlignedBit
|
||||
//(int(MatrixFlags)&int(DiagFlags)&AlignedBit),
|
||||
Flags = ((HereditaryBits|_LinearAccessMask|AlignedBit) & (unsigned int)(MatrixFlags)) | (_Vectorizable ? PacketAccessBit : 0)
|
||||
};
|
||||
|
||||
diagonal_product_evaluator_base(const MatrixType &mat, const DiagonalType &diag)
|
||||
|
||||
@@ -200,17 +200,82 @@ DenseBase<Derived>::reverse() const
|
||||
* In most cases it is probably better to simply use the reversed expression
|
||||
* of a matrix. However, when reversing the matrix data itself is really needed,
|
||||
* then this "in-place" version is probably the right choice because it provides
|
||||
* the following additional features:
|
||||
* the following additional benefits:
|
||||
* - less error prone: doing the same operation with .reverse() requires special care:
|
||||
* \code m = m.reverse().eval(); \endcode
|
||||
* - this API allows to avoid creating a temporary (the current implementation creates a temporary, but that could be avoided using swap)
|
||||
* - this API enables reverse operations without the need for a temporary
|
||||
* - it allows future optimizations (cache friendliness, etc.)
|
||||
*
|
||||
* \sa reverse() */
|
||||
* \sa VectorwiseOp::reverseInPlace(), reverse() */
|
||||
template<typename Derived>
|
||||
inline void DenseBase<Derived>::reverseInPlace()
|
||||
{
|
||||
derived() = derived().reverse().eval();
|
||||
if(cols()>rows())
|
||||
{
|
||||
Index half = cols()/2;
|
||||
leftCols(half).swap(rightCols(half).reverse());
|
||||
if((cols()%2)==1)
|
||||
{
|
||||
Index half2 = rows()/2;
|
||||
col(half).head(half2).swap(col(half).tail(half2).reverse());
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
Index half = rows()/2;
|
||||
topRows(half).swap(bottomRows(half).reverse());
|
||||
if((rows()%2)==1)
|
||||
{
|
||||
Index half2 = cols()/2;
|
||||
row(half).head(half2).swap(row(half).tail(half2).reverse());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
namespace internal {
|
||||
|
||||
template<int Direction>
|
||||
struct vectorwise_reverse_inplace_impl;
|
||||
|
||||
template<>
|
||||
struct vectorwise_reverse_inplace_impl<Vertical>
|
||||
{
|
||||
template<typename ExpressionType>
|
||||
static void run(ExpressionType &xpr)
|
||||
{
|
||||
Index half = xpr.rows()/2;
|
||||
xpr.topRows(half).swap(xpr.bottomRows(half).colwise().reverse());
|
||||
}
|
||||
};
|
||||
|
||||
template<>
|
||||
struct vectorwise_reverse_inplace_impl<Horizontal>
|
||||
{
|
||||
template<typename ExpressionType>
|
||||
static void run(ExpressionType &xpr)
|
||||
{
|
||||
Index half = xpr.cols()/2;
|
||||
xpr.leftCols(half).swap(xpr.rightCols(half).rowwise().reverse());
|
||||
}
|
||||
};
|
||||
|
||||
} // end namespace internal
|
||||
|
||||
/** This is the "in place" version of VectorwiseOp::reverse: it reverses each column or row of \c *this.
|
||||
*
|
||||
* In most cases it is probably better to simply use the reversed expression
|
||||
* of a matrix. However, when reversing the matrix data itself is really needed,
|
||||
* then this "in-place" version is probably the right choice because it provides
|
||||
* the following additional benefits:
|
||||
* - less error prone: doing the same operation with .reverse() requires special care:
|
||||
* \code m = m.reverse().eval(); \endcode
|
||||
* - this API enables reverse operations without the need for a temporary
|
||||
*
|
||||
* \sa DenseBase::reverseInPlace(), reverse() */
|
||||
template<typename ExpressionType, int Direction>
|
||||
void VectorwiseOp<ExpressionType,Direction>::reverseInPlace()
|
||||
{
|
||||
internal::vectorwise_reverse_inplace_impl<Direction>::run(_expression().const_cast_derived());
|
||||
}
|
||||
|
||||
} // end namespace Eigen
|
||||
|
||||
@@ -38,13 +38,17 @@ public:
|
||||
template<int StoreMode, int LoadMode>
|
||||
void assignPacket(Index row, Index col)
|
||||
{
|
||||
m_functor.template swapPacket<StoreMode,LoadMode,PacketScalar>(&m_dst.coeffRef(row,col), &const_cast<SrcEvaluatorTypeT&>(m_src).coeffRef(row,col));
|
||||
PacketScalar tmp = m_src.template packet<LoadMode>(row,col);
|
||||
const_cast<SrcEvaluatorTypeT&>(m_src).template writePacket<LoadMode>(row,col, m_dst.template packet<StoreMode>(row,col));
|
||||
m_dst.template writePacket<StoreMode>(row,col,tmp);
|
||||
}
|
||||
|
||||
template<int StoreMode, int LoadMode>
|
||||
void assignPacket(Index index)
|
||||
{
|
||||
m_functor.template swapPacket<StoreMode,LoadMode,PacketScalar>(&m_dst.coeffRef(index), &const_cast<SrcEvaluatorTypeT&>(m_src).coeffRef(index));
|
||||
PacketScalar tmp = m_src.template packet<LoadMode>(index);
|
||||
const_cast<SrcEvaluatorTypeT&>(m_src).template writePacket<LoadMode>(index, m_dst.template packet<StoreMode>(index));
|
||||
m_dst.template writePacket<StoreMode>(index,tmp);
|
||||
}
|
||||
|
||||
// TODO find a simple way not to have to copy/paste this function from generic_dense_assignment_kernel, by simple I mean no CRTP (Gael)
|
||||
|
||||
@@ -562,6 +562,8 @@ template<typename ExpressionType, int Direction> class VectorwiseOp
|
||||
void normalize() {
|
||||
m_matrix = this->normalized();
|
||||
}
|
||||
|
||||
inline void reverseInPlace();
|
||||
|
||||
/////////// Geometry module ///////////
|
||||
|
||||
|
||||
@@ -197,21 +197,21 @@ EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE double2 ploadt_ro<double2, Unaligned>(cons
|
||||
}
|
||||
#endif
|
||||
|
||||
template<> EIGEN_DEVICE_FUNC inline float4 pgather<float, float4>(const float* from, int stride) {
|
||||
template<> EIGEN_DEVICE_FUNC inline float4 pgather<float, float4>(const float* from, Index stride) {
|
||||
return make_float4(from[0*stride], from[1*stride], from[2*stride], from[3*stride]);
|
||||
}
|
||||
|
||||
template<> EIGEN_DEVICE_FUNC inline double2 pgather<double, double2>(const double* from, int stride) {
|
||||
template<> EIGEN_DEVICE_FUNC inline double2 pgather<double, double2>(const double* from, Index stride) {
|
||||
return make_double2(from[0*stride], from[1*stride]);
|
||||
}
|
||||
|
||||
template<> EIGEN_DEVICE_FUNC inline void pscatter<float, float4>(float* to, const float4& from, int stride) {
|
||||
template<> EIGEN_DEVICE_FUNC inline void pscatter<float, float4>(float* to, const float4& from, Index stride) {
|
||||
to[stride*0] = from.x;
|
||||
to[stride*1] = from.y;
|
||||
to[stride*2] = from.z;
|
||||
to[stride*3] = from.w;
|
||||
}
|
||||
template<> EIGEN_DEVICE_FUNC inline void pscatter<double, double2>(double* to, const double2& from, int stride) {
|
||||
template<> EIGEN_DEVICE_FUNC inline void pscatter<double, double2>(double* to, const double2& from, Index stride) {
|
||||
to[stride*0] = from.x;
|
||||
to[stride*1] = from.y;
|
||||
}
|
||||
@@ -245,14 +245,14 @@ template<> EIGEN_DEVICE_FUNC inline double predux_min<double2>(const double2& a)
|
||||
}
|
||||
|
||||
template<> EIGEN_DEVICE_FUNC inline float4 pabs<float4>(const float4& a) {
|
||||
return make_float4(fabs(a.x), fabs(a.y), fabs(a.z), fabs(a.w));
|
||||
return make_float4(fabsf(a.x), fabsf(a.y), fabsf(a.z), fabsf(a.w));
|
||||
}
|
||||
template<> EIGEN_DEVICE_FUNC inline double2 pabs<double2>(const double2& a) {
|
||||
return make_double2(abs(a.x), abs(a.y));
|
||||
return make_double2(fabs(a.x), fabs(a.y));
|
||||
}
|
||||
|
||||
|
||||
template<> EIGEN_DEVICE_FUNC inline void
|
||||
EIGEN_DEVICE_FUNC inline void
|
||||
ptranspose(PacketBlock<float4,4>& kernel) {
|
||||
double tmp = kernel.packet[0].y;
|
||||
kernel.packet[0].y = kernel.packet[1].x;
|
||||
@@ -279,7 +279,7 @@ ptranspose(PacketBlock<float4,4>& kernel) {
|
||||
kernel.packet[3].z = tmp;
|
||||
}
|
||||
|
||||
template<> EIGEN_DEVICE_FUNC inline void
|
||||
EIGEN_DEVICE_FUNC inline void
|
||||
ptranspose(PacketBlock<double2,2>& kernel) {
|
||||
double tmp = kernel.packet[0].y;
|
||||
kernel.packet[0].y = kernel.packet[1].x;
|
||||
|
||||
110
Eigen/src/Core/arch/NEON/BlockingSizesLookupTables.h
Normal file
110
Eigen/src/Core/arch/NEON/BlockingSizesLookupTables.h
Normal file
@@ -0,0 +1,110 @@
|
||||
// This file is part of Eigen, a lightweight C++ template library
|
||||
// for linear algebra.
|
||||
//
|
||||
// Copyright (C) 2015 Benoit Jacob <benoitjacob@google.com>
|
||||
//
|
||||
// This Source Code Form is subject to the terms of the Mozilla
|
||||
// Public License v. 2.0. If a copy of the MPL was not distributed
|
||||
// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
|
||||
#ifndef EIGEN_NEON_BLOCKING_SIZES_LOOKUP_TABLES_H
|
||||
#define EIGEN_NEON_BLOCKING_SIZES_LOOKUP_TABLES_H
|
||||
|
||||
namespace Eigen {
|
||||
namespace internal {
|
||||
|
||||
/* The following lookup table was generated from measurements on a Nexus 5,
|
||||
* which has a Qualcomm Krait 400 CPU. This is very representative of current
|
||||
* 32bit (ARMv7) Android devices. On the other hand, I don't know how
|
||||
* representative that is outside of these conditions. Accordingly,
|
||||
* let's only use this lookup table on ARM 32bit on Android for now.
|
||||
*
|
||||
* Measurements were single-threaded, with Scalar=float, compiled with
|
||||
* -mfpu=neon-vfpv4, so the pmadd instruction used was VFMA.F32.
|
||||
*
|
||||
* The device was cooled, allowing it to run a the max clock speed throughout.
|
||||
* This may not be representative of real-world thermal conditions.
|
||||
*
|
||||
* The benchmark attempted to flush caches to test cold-cache performance.
|
||||
*/
|
||||
#if EIGEN_ARCH_ARM && EIGEN_OS_ANDROID
|
||||
template<>
|
||||
struct BlockingSizesLookupTable<float, float> {
|
||||
static const size_t BaseSize = 16;
|
||||
static const size_t NumSizes = 8;
|
||||
static const unsigned short* Data() {
|
||||
static const unsigned short data[512] = {
|
||||
0x444, 0x445, 0x446, 0x447, 0x448, 0x449, 0x447, 0x447,
|
||||
0x454, 0x455, 0x456, 0x457, 0x458, 0x459, 0x45a, 0x456,
|
||||
0x464, 0x465, 0x466, 0x467, 0x468, 0x469, 0x46a, 0x467,
|
||||
0x474, 0x475, 0x476, 0x467, 0x478, 0x479, 0x476, 0x478,
|
||||
0x474, 0x475, 0x476, 0x477, 0x478, 0x479, 0x476, 0x476,
|
||||
0x474, 0x475, 0x476, 0x477, 0x478, 0x479, 0x496, 0x488,
|
||||
0x474, 0x475, 0x476, 0x4a6, 0x496, 0x496, 0x495, 0x4a6,
|
||||
0x474, 0x475, 0x466, 0x4a6, 0x497, 0x4a5, 0x496, 0x4a5,
|
||||
0x544, 0x545, 0x546, 0x547, 0x548, 0x549, 0x54a, 0x54b,
|
||||
0x554, 0x555, 0x556, 0x557, 0x558, 0x559, 0x55a, 0x55b,
|
||||
0x564, 0x565, 0x566, 0x567, 0x568, 0x569, 0x56a, 0x56b,
|
||||
0x564, 0x565, 0x566, 0x567, 0x568, 0x569, 0x56a, 0x576,
|
||||
0x564, 0x565, 0x566, 0x567, 0x568, 0x569, 0x56a, 0x587,
|
||||
0x564, 0x565, 0x566, 0x567, 0x596, 0x596, 0x596, 0x597,
|
||||
0x574, 0x565, 0x566, 0x596, 0x596, 0x5a6, 0x5a6, 0x5a6,
|
||||
0x564, 0x565, 0x5a6, 0x596, 0x5a6, 0x5a6, 0x5a6, 0x5a6,
|
||||
0x644, 0x645, 0x646, 0x647, 0x648, 0x649, 0x64a, 0x64b,
|
||||
0x644, 0x655, 0x656, 0x657, 0x658, 0x659, 0x65a, 0x65b,
|
||||
0x664, 0x665, 0x666, 0x667, 0x668, 0x669, 0x65a, 0x667,
|
||||
0x654, 0x665, 0x676, 0x677, 0x678, 0x679, 0x67a, 0x675,
|
||||
0x684, 0x675, 0x686, 0x687, 0x688, 0x688, 0x687, 0x686,
|
||||
0x664, 0x685, 0x666, 0x677, 0x697, 0x696, 0x697, 0x697,
|
||||
0x664, 0x665, 0x696, 0x696, 0x685, 0x6a6, 0x696, 0x696,
|
||||
0x664, 0x675, 0x686, 0x696, 0x6a6, 0x696, 0x696, 0x696,
|
||||
0x744, 0x745, 0x746, 0x747, 0x748, 0x749, 0x74a, 0x747,
|
||||
0x754, 0x755, 0x756, 0x757, 0x758, 0x759, 0x75a, 0x757,
|
||||
0x764, 0x765, 0x756, 0x767, 0x768, 0x759, 0x75a, 0x766,
|
||||
0x744, 0x755, 0x766, 0x777, 0x768, 0x759, 0x778, 0x777,
|
||||
0x744, 0x745, 0x766, 0x777, 0x788, 0x786, 0x786, 0x788,
|
||||
0x754, 0x755, 0x766, 0x787, 0x796, 0x796, 0x787, 0x796,
|
||||
0x684, 0x695, 0x696, 0x6a6, 0x795, 0x786, 0x795, 0x796,
|
||||
0x684, 0x695, 0x696, 0x795, 0x786, 0x796, 0x795, 0x796,
|
||||
0x844, 0x845, 0x846, 0x847, 0x848, 0x849, 0x848, 0x848,
|
||||
0x844, 0x855, 0x846, 0x847, 0x848, 0x849, 0x855, 0x857,
|
||||
0x844, 0x845, 0x846, 0x857, 0x848, 0x859, 0x866, 0x865,
|
||||
0x844, 0x855, 0x846, 0x847, 0x878, 0x859, 0x877, 0x877,
|
||||
0x844, 0x855, 0x846, 0x867, 0x886, 0x887, 0x885, 0x886,
|
||||
0x784, 0x785, 0x786, 0x877, 0x897, 0x885, 0x896, 0x896,
|
||||
0x684, 0x695, 0x686, 0x886, 0x885, 0x885, 0x886, 0x896,
|
||||
0x694, 0x6a5, 0x6a6, 0x885, 0x885, 0x886, 0x896, 0x896,
|
||||
0x944, 0x945, 0x946, 0x947, 0x948, 0x847, 0x847, 0x848,
|
||||
0x954, 0x855, 0x856, 0x947, 0x858, 0x857, 0x858, 0x858,
|
||||
0x944, 0x945, 0x946, 0x867, 0x948, 0x866, 0x867, 0x867,
|
||||
0x944, 0x975, 0x976, 0x877, 0x877, 0x877, 0x877, 0x877,
|
||||
0x784, 0x785, 0x886, 0x887, 0x886, 0x887, 0x887, 0x887,
|
||||
0x784, 0x785, 0x786, 0x796, 0x887, 0x897, 0x896, 0x896,
|
||||
0x684, 0x695, 0x6a6, 0x886, 0x886, 0x896, 0x896, 0x896,
|
||||
0x6a4, 0x6a5, 0x696, 0x896, 0x886, 0x896, 0x896, 0x896,
|
||||
0xa44, 0xa45, 0xa46, 0xa47, 0x847, 0x848, 0x847, 0x848,
|
||||
0xa44, 0xa45, 0x856, 0x857, 0x857, 0x857, 0x857, 0x857,
|
||||
0xa44, 0xa65, 0x866, 0x867, 0x867, 0x867, 0x867, 0x867,
|
||||
0x774, 0x875, 0x876, 0x877, 0x877, 0x877, 0x877, 0x877,
|
||||
0x784, 0x785, 0x886, 0x887, 0x887, 0x887, 0x887, 0x887,
|
||||
0x784, 0x785, 0x786, 0x787, 0x887, 0x896, 0x897, 0x897,
|
||||
0x684, 0x6a5, 0x696, 0x886, 0x886, 0x896, 0x896, 0x896,
|
||||
0x684, 0x6a5, 0x6a5, 0x886, 0x886, 0x896, 0x896, 0x896,
|
||||
0xb44, 0x845, 0x846, 0x847, 0x847, 0x945, 0x846, 0x946,
|
||||
0xb54, 0x855, 0x856, 0x857, 0x857, 0x856, 0x857, 0x856,
|
||||
0x864, 0x865, 0x866, 0x867, 0x867, 0x866, 0x866, 0x867,
|
||||
0x864, 0x875, 0x876, 0x877, 0x877, 0x877, 0x877, 0x877,
|
||||
0x784, 0x885, 0x886, 0x787, 0x887, 0x887, 0x887, 0x887,
|
||||
0x784, 0x785, 0x786, 0x796, 0x886, 0x897, 0x897, 0x897,
|
||||
0x684, 0x695, 0x696, 0x886, 0x896, 0x896, 0x896, 0x896,
|
||||
0x684, 0x685, 0x696, 0xb57, 0x896, 0x896, 0x896, 0x896
|
||||
};
|
||||
return data;
|
||||
}
|
||||
};
|
||||
#endif
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
#endif // EIGEN_NEON_BLOCKING_SIZES_LOOKUP_TABLES_H
|
||||
@@ -150,14 +150,6 @@ template<typename Scalar> struct swap_assign_op {
|
||||
swap(a,const_cast<Scalar&>(b));
|
||||
#endif
|
||||
}
|
||||
|
||||
template<int LhsAlignment, int RhsAlignment, typename Packet>
|
||||
EIGEN_STRONG_INLINE void swapPacket(Scalar* a, Scalar* b) const
|
||||
{
|
||||
Packet tmp = internal::ploadt<Packet,RhsAlignment>(b);
|
||||
internal::pstoret<Scalar,Packet,RhsAlignment>(b, internal::ploadt<Packet,LhsAlignment>(a));
|
||||
internal::pstoret<Scalar,Packet,LhsAlignment>(a, tmp);
|
||||
}
|
||||
};
|
||||
template<typename Scalar>
|
||||
struct functor_traits<swap_assign_op<Scalar> > {
|
||||
|
||||
@@ -25,21 +25,31 @@ inline std::ptrdiff_t manage_caching_sizes_helper(std::ptrdiff_t a, std::ptrdiff
|
||||
return a<=0 ? b : a;
|
||||
}
|
||||
|
||||
#if EIGEN_ARCH_i386_OR_x86_64
|
||||
const std::ptrdiff_t defaultL1CacheSize = 32*1024;
|
||||
const std::ptrdiff_t defaultL2CacheSize = 256*1024;
|
||||
const std::ptrdiff_t defaultL3CacheSize = 2*1024*1024;
|
||||
#else
|
||||
const std::ptrdiff_t defaultL1CacheSize = 16*1024;
|
||||
const std::ptrdiff_t defaultL2CacheSize = 512*1024;
|
||||
const std::ptrdiff_t defaultL3CacheSize = 512*1024;
|
||||
#endif
|
||||
|
||||
/** \internal */
|
||||
inline void manage_caching_sizes(Action action, std::ptrdiff_t* l1, std::ptrdiff_t* l2, std::ptrdiff_t* l3)
|
||||
{
|
||||
static bool m_cache_sizes_initialized = false;
|
||||
static std::ptrdiff_t m_l1CacheSize = 32*1024;
|
||||
static std::ptrdiff_t m_l2CacheSize = 256*1024;
|
||||
static std::ptrdiff_t m_l3CacheSize = 2*1024*1024;
|
||||
static std::ptrdiff_t m_l1CacheSize = 0;
|
||||
static std::ptrdiff_t m_l2CacheSize = 0;
|
||||
static std::ptrdiff_t m_l3CacheSize = 0;
|
||||
|
||||
if(!m_cache_sizes_initialized)
|
||||
{
|
||||
int l1CacheSize, l2CacheSize, l3CacheSize;
|
||||
queryCacheSizes(l1CacheSize, l2CacheSize, l3CacheSize);
|
||||
m_l1CacheSize = manage_caching_sizes_helper(l1CacheSize, 8*1024);
|
||||
m_l2CacheSize = manage_caching_sizes_helper(l2CacheSize, 256*1024);
|
||||
m_l3CacheSize = manage_caching_sizes_helper(l3CacheSize, 8*1024*1024);
|
||||
m_l1CacheSize = manage_caching_sizes_helper(l1CacheSize, defaultL1CacheSize);
|
||||
m_l2CacheSize = manage_caching_sizes_helper(l2CacheSize, defaultL2CacheSize);
|
||||
m_l3CacheSize = manage_caching_sizes_helper(l3CacheSize, defaultL3CacheSize);
|
||||
m_cache_sizes_initialized = true;
|
||||
}
|
||||
|
||||
@@ -64,45 +74,23 @@ inline void manage_caching_sizes(Action action, std::ptrdiff_t* l1, std::ptrdiff
|
||||
}
|
||||
}
|
||||
|
||||
/** \brief Computes the blocking parameters for a m x k times k x n matrix product
|
||||
*
|
||||
* \param[in,out] k Input: the third dimension of the product. Output: the blocking size along the same dimension.
|
||||
* \param[in,out] m Input: the number of rows of the left hand side. Output: the blocking size along the same dimension.
|
||||
* \param[in,out] n Input: the number of columns of the right hand side. Output: the blocking size along the same dimension.
|
||||
*
|
||||
* Given a m x k times k x n matrix product of scalar types \c LhsScalar and \c RhsScalar,
|
||||
* this function computes the blocking size parameters along the respective dimensions
|
||||
* for matrix products and related algorithms. The blocking sizes depends on various
|
||||
* parameters:
|
||||
* - the L1 and L2 cache sizes,
|
||||
* - the register level blocking sizes defined by gebp_traits,
|
||||
* - the number of scalars that fit into a packet (when vectorization is enabled).
|
||||
*
|
||||
* \sa setCpuCacheSizes */
|
||||
/* Helper for computeProductBlockingSizes.
|
||||
*
|
||||
* Given a m x k times k x n matrix product of scalar types \c LhsScalar and \c RhsScalar,
|
||||
* this function computes the blocking size parameters along the respective dimensions
|
||||
* for matrix products and related algorithms. The blocking sizes depends on various
|
||||
* parameters:
|
||||
* - the L1 and L2 cache sizes,
|
||||
* - the register level blocking sizes defined by gebp_traits,
|
||||
* - the number of scalars that fit into a packet (when vectorization is enabled).
|
||||
*
|
||||
* \sa setCpuCacheSizes */
|
||||
|
||||
template<typename LhsScalar, typename RhsScalar, int KcFactor>
|
||||
void computeProductBlockingSizes(Index& k, Index& m, Index& n, Index num_threads = 1)
|
||||
void evaluateProductBlockingSizesHeuristic(Index& k, Index& m, Index& n, Index num_threads = 1)
|
||||
{
|
||||
typedef gebp_traits<LhsScalar,RhsScalar> Traits;
|
||||
|
||||
#ifdef EIGEN_TEST_SPECIFIC_BLOCKING_SIZES
|
||||
if (EIGEN_TEST_SPECIFIC_BLOCKING_SIZES) {
|
||||
EIGEN_UNUSED_VARIABLE(num_threads);
|
||||
enum {
|
||||
kr = 8,
|
||||
mr = Traits::mr,
|
||||
nr = Traits::nr
|
||||
};
|
||||
k = std::min<Index>(k, EIGEN_TEST_SPECIFIC_BLOCKING_SIZE_K);
|
||||
if (k > kr) k -= k % kr;
|
||||
m = std::min<Index>(m, EIGEN_TEST_SPECIFIC_BLOCKING_SIZE_M);
|
||||
if (m > mr) m -= m % mr;
|
||||
n = std::min<Index>(n, EIGEN_TEST_SPECIFIC_BLOCKING_SIZE_N);
|
||||
if (n > nr) n -= n % nr;
|
||||
return;
|
||||
}
|
||||
#endif
|
||||
|
||||
// Explanations:
|
||||
// Let's recall that the product algorithms form mc x kc vertical panels A' on the lhs and
|
||||
// kc x nc blocks B' on the rhs. B' has to fit into L2/L3 cache. Moreover, A' is processed
|
||||
@@ -261,16 +249,69 @@ void computeProductBlockingSizes(Index& k, Index& m, Index& n, Index num_threads
|
||||
actual_lm = l2;
|
||||
max_mc = 576;
|
||||
}
|
||||
|
||||
Index mc = (std::min<Index>)(actual_lm/(3*k*sizeof(LhsScalar)), max_mc);
|
||||
if (mc > Traits::mr) mc -= mc % Traits::mr;
|
||||
|
||||
else if (mc==0) return;
|
||||
m = (m%mc)==0 ? mc
|
||||
: (mc - Traits::mr * ((mc/*-1*/-(m%mc))/(Traits::mr*(m/mc+1))));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
inline bool useSpecificBlockingSizes(Index& k, Index& m, Index& n)
|
||||
{
|
||||
#ifdef EIGEN_TEST_SPECIFIC_BLOCKING_SIZES
|
||||
if (EIGEN_TEST_SPECIFIC_BLOCKING_SIZES) {
|
||||
k = std::min<Index>(k, EIGEN_TEST_SPECIFIC_BLOCKING_SIZE_K);
|
||||
m = std::min<Index>(m, EIGEN_TEST_SPECIFIC_BLOCKING_SIZE_M);
|
||||
n = std::min<Index>(n, EIGEN_TEST_SPECIFIC_BLOCKING_SIZE_N);
|
||||
return true;
|
||||
}
|
||||
#else
|
||||
EIGEN_UNUSED_VARIABLE(k)
|
||||
EIGEN_UNUSED_VARIABLE(m)
|
||||
EIGEN_UNUSED_VARIABLE(n)
|
||||
#endif
|
||||
return false;
|
||||
}
|
||||
|
||||
/** \brief Computes the blocking parameters for a m x k times k x n matrix product
|
||||
*
|
||||
* \param[in,out] k Input: the third dimension of the product. Output: the blocking size along the same dimension.
|
||||
* \param[in,out] m Input: the number of rows of the left hand side. Output: the blocking size along the same dimension.
|
||||
* \param[in,out] n Input: the number of columns of the right hand side. Output: the blocking size along the same dimension.
|
||||
*
|
||||
* Given a m x k times k x n matrix product of scalar types \c LhsScalar and \c RhsScalar,
|
||||
* this function computes the blocking size parameters along the respective dimensions
|
||||
* for matrix products and related algorithms.
|
||||
*
|
||||
* The blocking size parameters may be evaluated:
|
||||
* - either by a heuristic based on cache sizes;
|
||||
* - or using a precomputed lookup table;
|
||||
* - or using fixed prescribed values (for testing purposes).
|
||||
*
|
||||
* \sa setCpuCacheSizes */
|
||||
|
||||
template<typename LhsScalar, typename RhsScalar, int KcFactor>
|
||||
void computeProductBlockingSizes(Index& k, Index& m, Index& n, Index num_threads = 1)
|
||||
{
|
||||
if (!useSpecificBlockingSizes(k, m, n)) {
|
||||
if (!lookupBlockingSizesFromTable<LhsScalar, RhsScalar>(k, m, n, num_threads)) {
|
||||
evaluateProductBlockingSizesHeuristic<LhsScalar, RhsScalar, KcFactor>(k, m, n, num_threads);
|
||||
}
|
||||
}
|
||||
|
||||
typedef gebp_traits<LhsScalar,RhsScalar> Traits;
|
||||
enum {
|
||||
kr = 8,
|
||||
mr = Traits::mr,
|
||||
nr = Traits::nr
|
||||
};
|
||||
if (k > kr) k -= k % kr;
|
||||
if (m > mr) m -= m % mr;
|
||||
if (n > nr) n -= n % nr;
|
||||
}
|
||||
|
||||
template<typename LhsScalar, typename RhsScalar>
|
||||
inline void computeProductBlockingSizes(Index& k, Index& m, Index& n, Index num_threads = 1)
|
||||
{
|
||||
@@ -339,11 +380,14 @@ public:
|
||||
nr = 4,
|
||||
|
||||
// register block size along the M direction (currently, this one cannot be modified)
|
||||
default_mr = (EIGEN_PLAIN_ENUM_MIN(16,NumberOfRegisters)/2/nr)*LhsPacketSize,
|
||||
#if defined(EIGEN_HAS_SINGLE_INSTRUCTION_MADD) && !defined(EIGEN_VECTORIZE_ALTIVEC) && !defined(EIGEN_VECTORIZE_VSX)
|
||||
// we assume 16 registers
|
||||
mr = 3*LhsPacketSize,
|
||||
// See bug 992, if the scalar type is not vectorizable but that EIGEN_HAS_SINGLE_INSTRUCTION_MADD is defined,
|
||||
// then using 3*LhsPacketSize triggers non-implemented paths in syrk.
|
||||
mr = Vectorizable ? 3*LhsPacketSize : default_mr,
|
||||
#else
|
||||
mr = (EIGEN_PLAIN_ENUM_MIN(16,NumberOfRegisters)/2/nr)*LhsPacketSize,
|
||||
mr = default_mr,
|
||||
#endif
|
||||
|
||||
LhsProgress = LhsPacketSize,
|
||||
@@ -974,12 +1018,11 @@ void gebp_kernel<LhsScalar,RhsScalar,Index,DataMapper,mr,nr,ConjugateLhs,Conjuga
|
||||
// Blocking sizes, i.e., 'depth' has been computed so that the micro horizontal panel of the lhs fit in L1.
|
||||
// However, if depth is too small, we can extend the number of rows of these horizontal panels.
|
||||
// This actual number of rows is computed as follow:
|
||||
const Index l1 = 32*1024; // in Bytes, TODO, l1 should be passed to this function.
|
||||
#ifdef EIGEN_TEST_SPECIFIC_BLOCKING_SIZES
|
||||
const Index l1 = defaultL1CacheSize; // in Bytes, TODO, l1 should be passed to this function.
|
||||
// The max(1, ...) here is needed because we may be using blocking params larger than what our known l1 cache size
|
||||
// suggests we should be using: either because our known l1 cache size is inaccurate (e.g. on Android, we can only guess),
|
||||
// or because we are testing specific blocking sizes.
|
||||
const Index actual_panel_rows = (3*LhsProgress) * std::max<Index>(1,( (l1 - sizeof(ResScalar)*mr*nr - depth*nr*sizeof(RhsScalar)) / (depth * sizeof(LhsScalar) * 3*LhsProgress) ));
|
||||
#else
|
||||
const Index actual_panel_rows = (3*LhsProgress) * ( (l1 - sizeof(ResScalar)*mr*nr - depth*nr*sizeof(RhsScalar)) / (depth * sizeof(LhsScalar) * 3*LhsProgress) );
|
||||
#endif
|
||||
for(Index i1=0; i1<peeled_mc3; i1+=actual_panel_rows)
|
||||
{
|
||||
const Index actual_panel_end = (std::min)(i1+actual_panel_rows, peeled_mc3);
|
||||
@@ -1211,12 +1254,12 @@ void gebp_kernel<LhsScalar,RhsScalar,Index,DataMapper,mr,nr,ConjugateLhs,Conjuga
|
||||
//---------- Process 2 * LhsProgress rows at once ----------
|
||||
if(mr>=2*Traits::LhsProgress)
|
||||
{
|
||||
const Index l1 = 32*1024; // in Bytes, TODO, l1 should be passed to this function.
|
||||
#ifdef EIGEN_TEST_SPECIFIC_BLOCKING_SIZES
|
||||
const Index l1 = defaultL1CacheSize; // in Bytes, TODO, l1 should be passed to this function.
|
||||
// The max(1, ...) here is needed because we may be using blocking params larger than what our known l1 cache size
|
||||
// suggests we should be using: either because our known l1 cache size is inaccurate (e.g. on Android, we can only guess),
|
||||
// or because we are testing specific blocking sizes.
|
||||
Index actual_panel_rows = (2*LhsProgress) * std::max<Index>(1,( (l1 - sizeof(ResScalar)*mr*nr - depth*nr*sizeof(RhsScalar)) / (depth * sizeof(LhsScalar) * 2*LhsProgress) ));
|
||||
#else
|
||||
Index actual_panel_rows = (2*LhsProgress) * ( (l1 - sizeof(ResScalar)*mr*nr - depth*nr*sizeof(RhsScalar)) / (depth * sizeof(LhsScalar) * 2*LhsProgress) );
|
||||
#endif
|
||||
|
||||
for(Index i1=peeled_mc3; i1<peeled_mc2; i1+=actual_panel_rows)
|
||||
{
|
||||
Index actual_panel_end = (std::min)(i1+actual_panel_rows, peeled_mc2);
|
||||
|
||||
97
Eigen/src/Core/products/LookupBlockingSizesTable.h
Normal file
97
Eigen/src/Core/products/LookupBlockingSizesTable.h
Normal file
@@ -0,0 +1,97 @@
|
||||
// This file is part of Eigen, a lightweight C++ template library
|
||||
// for linear algebra.
|
||||
//
|
||||
// Copyright (C) 2015 Benoit Jacob <benoitjacob@google.com>
|
||||
//
|
||||
// This Source Code Form is subject to the terms of the Mozilla
|
||||
// Public License v. 2.0. If a copy of the MPL was not distributed
|
||||
// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
|
||||
#ifndef EIGEN_LOOKUP_BLOCKING_SIZES_TABLE_H
|
||||
#define EIGEN_LOOKUP_BLOCKING_SIZES_TABLE_H
|
||||
|
||||
namespace Eigen {
|
||||
|
||||
namespace internal {
|
||||
|
||||
template <typename LhsScalar,
|
||||
typename RhsScalar,
|
||||
bool HasLookupTable = BlockingSizesLookupTable<LhsScalar, RhsScalar>::NumSizes != 0 >
|
||||
struct LookupBlockingSizesFromTableImpl
|
||||
{
|
||||
static bool run(Index&, Index&, Index&, Index)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
};
|
||||
|
||||
inline size_t floor_log2_helper(unsigned short& x, size_t offset)
|
||||
{
|
||||
unsigned short y = x >> offset;
|
||||
if (y) {
|
||||
x = y;
|
||||
return offset;
|
||||
} else {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
inline size_t floor_log2(unsigned short x)
|
||||
{
|
||||
return floor_log2_helper(x, 8)
|
||||
+ floor_log2_helper(x, 4)
|
||||
+ floor_log2_helper(x, 2)
|
||||
+ floor_log2_helper(x, 1);
|
||||
}
|
||||
|
||||
inline size_t ceil_log2(unsigned short x)
|
||||
{
|
||||
return x > 1 ? floor_log2(x - 1) + 1 : 0;
|
||||
}
|
||||
|
||||
template <typename LhsScalar,
|
||||
typename RhsScalar>
|
||||
struct LookupBlockingSizesFromTableImpl<LhsScalar, RhsScalar, true>
|
||||
{
|
||||
static bool run(Index& k, Index& m, Index& n, Index)
|
||||
{
|
||||
using std::min;
|
||||
using std::max;
|
||||
typedef BlockingSizesLookupTable<LhsScalar, RhsScalar> Table;
|
||||
const unsigned short minsize = Table::BaseSize;
|
||||
const unsigned short maxsize = minsize << (Table::NumSizes - 1);
|
||||
const unsigned short k_clamped = max<unsigned short>(minsize, min<Index>(k, maxsize));
|
||||
const unsigned short m_clamped = max<unsigned short>(minsize, min<Index>(m, maxsize));
|
||||
const unsigned short n_clamped = max<unsigned short>(minsize, min<Index>(n, maxsize));
|
||||
const size_t k_index = ceil_log2(k_clamped / minsize);
|
||||
const size_t m_index = ceil_log2(m_clamped / minsize);
|
||||
const size_t n_index = ceil_log2(n_clamped / minsize);
|
||||
const size_t index = n_index + Table::NumSizes * (m_index + Table::NumSizes * k_index);
|
||||
const unsigned short table_entry = Table::Data()[index];
|
||||
k = min<Index>(k, 1 << ((table_entry & 0xf00) >> 8));
|
||||
m = min<Index>(m, 1 << ((table_entry & 0x0f0) >> 4));
|
||||
n = min<Index>(n, 1 << ((table_entry & 0x00f) >> 0));
|
||||
return true;
|
||||
}
|
||||
};
|
||||
|
||||
template <typename LhsScalar,
|
||||
typename RhsScalar>
|
||||
bool lookupBlockingSizesFromTable(Index& k, Index& m, Index& n, Index num_threads)
|
||||
{
|
||||
if (num_threads > 1) {
|
||||
// We don't currently have lookup tables recorded for multithread performance,
|
||||
// and we have confirmed experimentally that our single-thread-recorded LUTs are
|
||||
// poor for multithread performance, and our LUTs don't currently contain
|
||||
// any annotation about multithread status (FIXME - we need that).
|
||||
// So for now, we just early-return here.
|
||||
return false;
|
||||
}
|
||||
return LookupBlockingSizesFromTableImpl<LhsScalar, RhsScalar>::run(k, m, n, num_threads);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
#endif // EIGEN_LOOKUP_BLOCKING_SIZES_TABLE_H
|
||||
@@ -214,7 +214,7 @@ class blas_data_mapper {
|
||||
}
|
||||
|
||||
template<typename SubPacket>
|
||||
EIGEN_ALWAYS_INLINE void scatterPacket(Index i, Index j, SubPacket p) const {
|
||||
EIGEN_ALWAYS_INLINE void scatterPacket(Index i, Index j, const SubPacket &p) const {
|
||||
pscatter<Scalar, SubPacket>(&operator()(i, j), p, m_stride);
|
||||
}
|
||||
|
||||
|
||||
@@ -287,6 +287,14 @@ struct stem_function
|
||||
typedef std::complex<typename NumTraits<Scalar>::Real> ComplexScalar;
|
||||
typedef ComplexScalar type(ComplexScalar, int);
|
||||
};
|
||||
|
||||
template <typename LhsScalar,
|
||||
typename RhsScalar>
|
||||
struct BlockingSizesLookupTable
|
||||
{
|
||||
static const size_t NumSizes = 0;
|
||||
};
|
||||
|
||||
}
|
||||
|
||||
} // end namespace Eigen
|
||||
|
||||
@@ -213,7 +213,8 @@
|
||||
#endif
|
||||
|
||||
/// \internal EIGEN_OS_ANDROID set to 1 if the OS is Android
|
||||
#if defined(__ANDROID__)
|
||||
// note: ANDROID is defined when using ndk_build, __ANDROID__ is defined when using a standalone toolchain.
|
||||
#if defined(__ANDROID__) || defined(ANDROID)
|
||||
#define EIGEN_OS_ANDROID 1
|
||||
#else
|
||||
#define EIGEN_OS_ANDROID 0
|
||||
@@ -318,6 +319,9 @@
|
||||
// Defined the boundary (in bytes) on which the data needs to be aligned. Note
|
||||
// that unless EIGEN_ALIGN is defined and not equal to 0, the data may not be
|
||||
// aligned at all regardless of the value of this #define.
|
||||
// TODO should be renamed EIGEN_MAXIMAL_ALIGN_BYTES,
|
||||
// for instance with AVX 1 EIGEN_MAXIMAL_ALIGN_BYTES=32 while for 'int' 16 bytes alignment is always enough,
|
||||
// and 16 bytes alignment is also enough for Vector4f.
|
||||
#define EIGEN_ALIGN_BYTES 16
|
||||
|
||||
#ifdef EIGEN_DONT_ALIGN
|
||||
|
||||
@@ -159,13 +159,16 @@ class compute_matrix_evaluator_flags
|
||||
enum {
|
||||
row_major_bit = Options&RowMajor ? RowMajorBit : 0,
|
||||
is_dynamic_size_storage = MaxRows==Dynamic || MaxCols==Dynamic,
|
||||
|
||||
// TODO: should check for smaller packet types once we can handle multi-sized packet types
|
||||
align_bytes = int(packet_traits<Scalar>::size) * sizeof(Scalar),
|
||||
|
||||
aligned_bit =
|
||||
(
|
||||
((Options&DontAlign)==0)
|
||||
&& (
|
||||
#if EIGEN_ALIGN_STATICALLY
|
||||
((!is_dynamic_size_storage) && (((MaxCols*MaxRows*int(sizeof(Scalar))) % EIGEN_ALIGN_BYTES) == 0))
|
||||
((!is_dynamic_size_storage) && (((MaxCols*MaxRows*int(sizeof(Scalar))) % align_bytes) == 0))
|
||||
#else
|
||||
0
|
||||
#endif
|
||||
|
||||
Reference in New Issue
Block a user