mirror of
https://gitlab.com/libeigen/eigen.git
synced 2026-04-10 11:34:33 +08:00
* refactoring of Product:
* use ProductReturnType<>::Type to get the correct Product xpr type * Product is no longer instanciated for xpr types which are evaluated * vectorization of "a.transpose() * b" for the normal product (small and fixed-size matrix) * some cleanning * removed ArrayBase
This commit is contained in:
Gael Guennebaud
@@ -26,123 +26,69 @@
|
||||
#ifndef EIGEN_PRODUCT_H
|
||||
#define EIGEN_PRODUCT_H
|
||||
|
||||
template<int Index, int Size, typename Lhs, typename Rhs>
|
||||
struct ei_product_impl
|
||||
{
|
||||
inline static void run(int row, int col, const Lhs& lhs, const Rhs& rhs,
|
||||
typename Lhs::Scalar &res)
|
||||
{
|
||||
ei_product_impl<Index-1, Size, Lhs, Rhs>::run(row, col, lhs, rhs, res);
|
||||
res += lhs.coeff(row, Index) * rhs.coeff(Index, col);
|
||||
}
|
||||
/***************************
|
||||
*** Forward declarations ***
|
||||
***************************/
|
||||
|
||||
enum {
|
||||
ColMajorProduct,
|
||||
RowMajorProduct
|
||||
};
|
||||
|
||||
template<int Size, typename Lhs, typename Rhs>
|
||||
struct ei_product_impl<0, Size, Lhs, Rhs>
|
||||
{
|
||||
inline static void run(int row, int col, const Lhs& lhs, const Rhs& rhs,
|
||||
typename Lhs::Scalar &res)
|
||||
{
|
||||
res = lhs.coeff(row, 0) * rhs.coeff(0, col);
|
||||
}
|
||||
};
|
||||
template<int VectorizationMode, int Index, typename Lhs, typename Rhs>
|
||||
struct ei_product_coeff_impl;
|
||||
|
||||
template<typename Lhs, typename Rhs>
|
||||
struct ei_product_impl<Dynamic, Dynamic, Lhs, Rhs>
|
||||
{
|
||||
inline static void run(int row, int col, const Lhs& lhs, const Rhs& rhs, typename Lhs::Scalar& res)
|
||||
{
|
||||
res = lhs.coeff(row, 0) * rhs.coeff(0, col);
|
||||
for(int i = 1; i < lhs.cols(); i++)
|
||||
res += lhs.coeff(row, i) * rhs.coeff(i, col);
|
||||
}
|
||||
};
|
||||
template<int StorageOrder, int Index, typename Lhs, typename Rhs, typename PacketScalar>
|
||||
struct ei_product_packet_impl;
|
||||
|
||||
// prevent buggy user code from causing an infinite recursion
|
||||
template<int Index, typename Lhs, typename Rhs>
|
||||
struct ei_product_impl<Index, 0, Lhs, Rhs>
|
||||
{
|
||||
inline static void run(int, int, const Lhs&, const Rhs&, typename Lhs::Scalar&) {}
|
||||
};
|
||||
template<typename T> class ei_product_eval_to_column_major;
|
||||
|
||||
//----------
|
||||
|
||||
template<bool RowMajor, int Index, int Size, typename Lhs, typename Rhs, typename PacketScalar>
|
||||
struct ei_packet_product_impl;
|
||||
|
||||
template<int Index, int Size, typename Lhs, typename Rhs, typename PacketScalar>
|
||||
struct ei_packet_product_impl<true, Index, Size, Lhs, Rhs, PacketScalar>
|
||||
{
|
||||
inline static void run(int row, int col, const Lhs& lhs, const Rhs& rhs, PacketScalar &res)
|
||||
{
|
||||
ei_packet_product_impl<true, Index-1, Size, Lhs, Rhs, PacketScalar>::run(row, col, lhs, rhs, res);
|
||||
res = ei_pmadd(ei_pset1(lhs.coeff(row, Index)), rhs.template packet<Aligned>(Index, col), res);
|
||||
}
|
||||
};
|
||||
|
||||
template<int Index, int Size, typename Lhs, typename Rhs, typename PacketScalar>
|
||||
struct ei_packet_product_impl<false, Index, Size, Lhs, Rhs, PacketScalar>
|
||||
{
|
||||
inline static void run(int row, int col, const Lhs& lhs, const Rhs& rhs, PacketScalar &res)
|
||||
{
|
||||
ei_packet_product_impl<false, Index-1, Size, Lhs, Rhs, PacketScalar>::run(row, col, lhs, rhs, res);
|
||||
res = ei_pmadd(lhs.template packet<Aligned>(row, Index), ei_pset1(rhs.coeff(Index, col)), res);
|
||||
}
|
||||
};
|
||||
|
||||
template<int Size, typename Lhs, typename Rhs, typename PacketScalar>
|
||||
struct ei_packet_product_impl<true, 0, Size, Lhs, Rhs, PacketScalar>
|
||||
{
|
||||
inline static void run(int row, int col, const Lhs& lhs, const Rhs& rhs, PacketScalar &res)
|
||||
{
|
||||
res = ei_pmul(ei_pset1(lhs.coeff(row, 0)),rhs.template packet<Aligned>(0, col));
|
||||
}
|
||||
};
|
||||
|
||||
template<int Size, typename Lhs, typename Rhs, typename PacketScalar>
|
||||
struct ei_packet_product_impl<false, 0, Size, Lhs, Rhs, PacketScalar>
|
||||
{
|
||||
inline static void run(int row, int col, const Lhs& lhs, const Rhs& rhs, PacketScalar &res)
|
||||
{
|
||||
res = ei_pmul(lhs.template packet<Aligned>(row, 0), ei_pset1(rhs.coeff(0, col)));
|
||||
}
|
||||
};
|
||||
|
||||
template<bool RowMajor, int Index, typename Lhs, typename Rhs, typename PacketScalar>
|
||||
struct ei_packet_product_impl<RowMajor, Index, Dynamic, Lhs, Rhs, PacketScalar>
|
||||
{
|
||||
inline static void run(int row, int col, const Lhs& lhs, const Rhs& rhs, PacketScalar& res)
|
||||
{
|
||||
res = ei_pmul(ei_pset1(lhs.coeff(row, 0)),rhs.template packet<Aligned>(0, col));
|
||||
for(int i = 1; i < lhs.cols(); i++)
|
||||
res = ei_pmadd(ei_pset1(lhs.coeff(row, i)), rhs.template packet<Aligned>(i, col), res);
|
||||
}
|
||||
};
|
||||
|
||||
template<int Index, typename Lhs, typename Rhs, typename PacketScalar>
|
||||
struct ei_packet_product_impl<false, Index, Dynamic, Lhs, Rhs, PacketScalar>
|
||||
{
|
||||
inline static void run(int row, int col, const Lhs& lhs, const Rhs& rhs, PacketScalar& res)
|
||||
{
|
||||
res = ei_pmul(lhs.template packet<Aligned>(row, 0), ei_pset1(rhs.coeff(0, col)));
|
||||
for(int i = 1; i < lhs.cols(); i++)
|
||||
res = ei_pmadd(lhs.template packet<Aligned>(row, i), ei_pset1(rhs.coeff(i, col)), res);
|
||||
}
|
||||
};
|
||||
|
||||
/** \class Product
|
||||
/** \class ProductReturnType
|
||||
*
|
||||
* \brief Expression of the product of two matrices
|
||||
* \brief Helper class to get the correct and optimized returned type of operator*
|
||||
*
|
||||
* \param Lhs the type of the left-hand side
|
||||
* \param Rhs the type of the right-hand side
|
||||
* \param EvalMode internal use only
|
||||
* \param ProductMode the type of the product (determined automatically by ei_product_mode)
|
||||
*
|
||||
* This class represents an expression of the product of two matrices.
|
||||
* It is the return type of the operator* between matrices, and most of the time
|
||||
* this is the only way it is used.
|
||||
* This class defines the typename Type representing the optimized product expression
|
||||
* between two matrix expressions. In practice, using ProductReturnType<Lhs,Rhs>::Type
|
||||
* is the recommended way to define the result type of a function returning an expression
|
||||
* which involve a matrix product. The class Product or DiagonalProduct should never be
|
||||
* used directly.
|
||||
*
|
||||
* \sa class Product, class DiagonalProduct, MatrixBase::operator*(const MatrixBase<OtherDerived>&)
|
||||
*/
|
||||
template<typename Lhs, typename Rhs> struct ei_product_eval_mode
|
||||
template<typename Lhs, typename Rhs, int ProductMode>
|
||||
struct ProductReturnType
|
||||
{
|
||||
typedef typename ei_nested<Lhs,Rhs::ColsAtCompileTime>::type LhsNested;
|
||||
typedef typename ei_nested<Rhs,Lhs::RowsAtCompileTime>::type RhsNested;
|
||||
|
||||
typedef Product<typename ei_unconst<LhsNested>::type,
|
||||
typename ei_unconst<RhsNested>::type, ProductMode> Type;
|
||||
};
|
||||
|
||||
// cache friendly specialization
|
||||
template<typename Lhs, typename Rhs>
|
||||
struct ProductReturnType<Lhs,Rhs,CacheFriendlyProduct>
|
||||
{
|
||||
typedef typename ei_nested<Lhs,Rhs::ColsAtCompileTime>::type LhsNested;
|
||||
|
||||
typedef typename ei_nested<Rhs,Lhs::RowsAtCompileTime,
|
||||
typename ei_product_eval_to_column_major<Rhs>::type
|
||||
>::type RhsNested;
|
||||
|
||||
typedef Product<typename ei_unconst<LhsNested>::type,
|
||||
typename ei_unconst<RhsNested>::type, CacheFriendlyProduct> Type;
|
||||
};
|
||||
|
||||
/* Helper class to determine the type of the product, can be either:
|
||||
* - NormalProduct
|
||||
* - CacheFriendlyProduct
|
||||
* - NormalProduct
|
||||
*/
|
||||
template<typename Lhs, typename Rhs> struct ei_product_mode
|
||||
{
|
||||
enum{ value = ((Rhs::Flags&Diagonal)==Diagonal) || ((Lhs::Flags&Diagonal)==Diagonal)
|
||||
? DiagonalProduct
|
||||
@@ -152,100 +98,103 @@ template<typename Lhs, typename Rhs> struct ei_product_eval_mode
|
||||
? CacheFriendlyProduct : NormalProduct };
|
||||
};
|
||||
|
||||
template<typename T> class ei_product_eval_to_column_major
|
||||
/** \class Product
|
||||
*
|
||||
* \brief Expression of the product of two matrices
|
||||
*
|
||||
* \param LhsNested the type used to store the left-hand side
|
||||
* \param RhsNested the type used to store the right-hand side
|
||||
* \param ProductMode the type of the product
|
||||
*
|
||||
* This class represents an expression of the product of two matrices.
|
||||
* It is the return type of the operator* between matrices. Its template
|
||||
* arguments are determined automatically by ProductReturnType. Therefore,
|
||||
* Product should be used direclty. To determine the result type of a function
|
||||
* which involve a matrix product, use ProductReturnType::Type.
|
||||
*
|
||||
* \sa ProductReturnType, MatrixBase::operator*(const MatrixBase<OtherDerived>&)
|
||||
*/
|
||||
template<typename LhsNested, typename RhsNested, int ProductMode>
|
||||
struct ei_traits<Product<LhsNested, RhsNested, ProductMode> >
|
||||
{
|
||||
typedef typename ei_traits<T>::Scalar _Scalar;
|
||||
enum {
|
||||
_Rows = ei_traits<T>::RowsAtCompileTime,
|
||||
_Cols = ei_traits<T>::ColsAtCompileTime,
|
||||
_MaxRows = ei_traits<T>::MaxRowsAtCompileTime,
|
||||
_MaxCols = ei_traits<T>::MaxColsAtCompileTime,
|
||||
_Flags = ei_traits<T>::Flags
|
||||
};
|
||||
|
||||
public:
|
||||
typedef Matrix<_Scalar,
|
||||
_Rows, _Cols, _MaxRows, _MaxCols,
|
||||
ei_corrected_matrix_flags<
|
||||
_Scalar,
|
||||
_Rows, _Cols, _MaxRows, _MaxCols,
|
||||
_Flags
|
||||
>::ret & ~RowMajorBit
|
||||
> type;
|
||||
};
|
||||
|
||||
// as ei_nested, but evaluate to a column-major matrix if an evaluation is required
|
||||
template<typename T, int n=1> struct ei_product_nested_rhs
|
||||
{
|
||||
typedef typename ei_meta_if<
|
||||
ei_must_nest_by_value<T>::ret,
|
||||
T,
|
||||
typename ei_meta_if<
|
||||
((ei_traits<T>::Flags & EvalBeforeNestingBit)
|
||||
|| (n+1) * (NumTraits<typename ei_traits<T>::Scalar>::ReadCost) < (n-1) * T::CoeffReadCost),
|
||||
typename ei_product_eval_to_column_major<T>::type,
|
||||
const T&
|
||||
>::ret
|
||||
>::ret type;
|
||||
};
|
||||
|
||||
template<typename Lhs, typename Rhs, int EvalMode>
|
||||
struct ei_traits<Product<Lhs, Rhs, EvalMode> >
|
||||
{
|
||||
typedef typename Lhs::Scalar Scalar;
|
||||
typedef typename ei_nested<Lhs,Rhs::ColsAtCompileTime>::type LhsNested;
|
||||
typedef typename ei_meta_if<EvalMode==CacheFriendlyProduct,
|
||||
typename ei_product_nested_rhs<Rhs,Lhs::RowsAtCompileTime>::type,
|
||||
typename ei_nested<Rhs,Lhs::RowsAtCompileTime>::type>::ret RhsNested;
|
||||
// clean the nested types:
|
||||
typedef typename ei_unconst<typename ei_unref<LhsNested>::type>::type _LhsNested;
|
||||
typedef typename ei_unconst<typename ei_unref<RhsNested>::type>::type _RhsNested;
|
||||
typedef typename _LhsNested::Scalar Scalar;
|
||||
|
||||
enum {
|
||||
LhsCoeffReadCost = _LhsNested::CoeffReadCost,
|
||||
RhsCoeffReadCost = _RhsNested::CoeffReadCost,
|
||||
LhsFlags = _LhsNested::Flags,
|
||||
RhsFlags = _RhsNested::Flags,
|
||||
RowsAtCompileTime = Lhs::RowsAtCompileTime,
|
||||
ColsAtCompileTime = Rhs::ColsAtCompileTime,
|
||||
MaxRowsAtCompileTime = Lhs::MaxRowsAtCompileTime,
|
||||
MaxColsAtCompileTime = Rhs::MaxColsAtCompileTime,
|
||||
// the vectorization flags are only used by the normal product,
|
||||
// the other one is always vectorized !
|
||||
_RhsPacketAccess = (RhsFlags & RowMajorBit) && (RhsFlags & PacketAccessBit) && (ColsAtCompileTime % ei_packet_traits<Scalar>::size == 0),
|
||||
_LhsPacketAccess = (!(LhsFlags & RowMajorBit)) && (LhsFlags & PacketAccessBit) && (RowsAtCompileTime % ei_packet_traits<Scalar>::size == 0),
|
||||
_PacketAccess = (_LhsPacketAccess || _RhsPacketAccess) ? 1 : 0,
|
||||
_RowMajor = (RhsFlags & RowMajorBit)
|
||||
&& (EvalMode==(int)CacheFriendlyProduct ? (int)LhsFlags & RowMajorBit : (!_LhsPacketAccess)),
|
||||
_LostBits = ~((_RowMajor ? 0 : RowMajorBit)
|
||||
|
||||
RowsAtCompileTime = _LhsNested::RowsAtCompileTime,
|
||||
ColsAtCompileTime = _RhsNested::ColsAtCompileTime,
|
||||
InnerSize = EIGEN_ENUM_MIN(_LhsNested::ColsAtCompileTime, _RhsNested::RowsAtCompileTime),
|
||||
|
||||
MaxRowsAtCompileTime = _LhsNested::MaxRowsAtCompileTime,
|
||||
MaxColsAtCompileTime = _RhsNested::MaxColsAtCompileTime,
|
||||
|
||||
LhsRowMajor = LhsFlags & RowMajorBit,
|
||||
RhsRowMajor = RhsFlags & RowMajorBit,
|
||||
|
||||
CanVectorizeRhs = RhsRowMajor && (RhsFlags & PacketAccessBit)
|
||||
&& (ColsAtCompileTime % ei_packet_traits<Scalar>::size == 0),
|
||||
|
||||
CanVectorizeLhs = (!LhsRowMajor) && (LhsFlags & PacketAccessBit)
|
||||
&& (RowsAtCompileTime % ei_packet_traits<Scalar>::size == 0),
|
||||
|
||||
CanVectorizeInner = LhsRowMajor && (!RhsRowMajor) && (LhsFlags & PacketAccessBit) && (RhsFlags & PacketAccessBit)
|
||||
&& (InnerSize!=Dynamic) && (InnerSize % ei_packet_traits<Scalar>::size == 0),
|
||||
|
||||
EvalToRowMajor = (RhsFlags & RowMajorBit)
|
||||
&& (ProductMode==(int)CacheFriendlyProduct ? (int)LhsFlags & RowMajorBit : (!CanVectorizeLhs)),
|
||||
|
||||
RemovedBits = ~((EvalToRowMajor ? 0 : RowMajorBit)
|
||||
| ((RowsAtCompileTime == Dynamic || ColsAtCompileTime == Dynamic) ? 0 : LargeBit)
|
||||
| LinearAccessBit),
|
||||
Flags = ((unsigned int)(LhsFlags | RhsFlags) & HereditaryBits & _LostBits)
|
||||
|
||||
Flags = ((unsigned int)(LhsFlags | RhsFlags) & HereditaryBits & RemovedBits)
|
||||
| EvalBeforeAssigningBit
|
||||
| EvalBeforeNestingBit
|
||||
| (_PacketAccess ? PacketAccessBit : 0),
|
||||
CoeffReadCost
|
||||
= Lhs::ColsAtCompileTime == Dynamic
|
||||
? Dynamic
|
||||
: Lhs::ColsAtCompileTime
|
||||
* (NumTraits<Scalar>::MulCost + LhsCoeffReadCost + RhsCoeffReadCost)
|
||||
+ (Lhs::ColsAtCompileTime - 1) * NumTraits<Scalar>::AddCost
|
||||
| (CanVectorizeLhs || CanVectorizeRhs ? PacketAccessBit : 0),
|
||||
|
||||
CoeffReadCost = InnerSize == Dynamic ? Dynamic
|
||||
: InnerSize * (NumTraits<Scalar>::MulCost + LhsCoeffReadCost + RhsCoeffReadCost)
|
||||
+ (InnerSize - 1) * NumTraits<Scalar>::AddCost
|
||||
};
|
||||
};
|
||||
|
||||
template<typename Lhs, typename Rhs, int EvalMode> class Product : ei_no_assignment_operator,
|
||||
public MatrixBase<Product<Lhs, Rhs, EvalMode> >
|
||||
template<typename LhsNested, typename RhsNested, int ProductMode> class Product : ei_no_assignment_operator,
|
||||
public MatrixBase<Product<LhsNested, RhsNested, ProductMode> >
|
||||
{
|
||||
public:
|
||||
|
||||
EIGEN_GENERIC_PUBLIC_INTERFACE(Product)
|
||||
typedef typename ei_traits<Product>::LhsNested LhsNested;
|
||||
typedef typename ei_traits<Product>::RhsNested RhsNested;
|
||||
|
||||
private:
|
||||
|
||||
typedef typename ei_traits<Product>::_LhsNested _LhsNested;
|
||||
typedef typename ei_traits<Product>::_RhsNested _RhsNested;
|
||||
|
||||
enum {
|
||||
PacketSize = ei_packet_traits<Scalar>::size
|
||||
PacketSize = ei_packet_traits<Scalar>::size,
|
||||
InnerSize = ei_traits<Product>::InnerSize,
|
||||
Unroll = CoeffReadCost <= EIGEN_UNROLLING_LIMIT,
|
||||
CanVectorizeInner = ei_traits<Product>::CanVectorizeInner && Unroll
|
||||
};
|
||||
|
||||
typedef ei_product_coeff_impl<CanVectorizeInner ? InnerVectorization : NoVectorization,
|
||||
Unroll ? InnerSize-1 : Dynamic,
|
||||
_LhsNested, _RhsNested> ScalarCoeffImpl;
|
||||
|
||||
typedef ei_product_packet_impl<Flags&RowMajorBit ? RowMajorProduct : ColMajorProduct,
|
||||
Unroll ? InnerSize-1 : Dynamic,
|
||||
_LhsNested, _RhsNested, PacketScalar> PacketCoeffImpl;
|
||||
|
||||
public:
|
||||
|
||||
template<typename Lhs, typename Rhs>
|
||||
inline Product(const Lhs& lhs, const Rhs& rhs)
|
||||
: m_lhs(lhs), m_rhs(rhs)
|
||||
{
|
||||
@@ -268,23 +217,15 @@ template<typename Lhs, typename Rhs, int EvalMode> class Product : ei_no_assignm
|
||||
const Scalar _coeff(int row, int col) const
|
||||
{
|
||||
Scalar res;
|
||||
const bool unroll = CoeffReadCost <= EIGEN_UNROLLING_LIMIT;
|
||||
ei_product_impl<unroll ? Lhs::ColsAtCompileTime-1 : Dynamic,
|
||||
unroll ? Lhs::ColsAtCompileTime : Dynamic,
|
||||
_LhsNested, _RhsNested>
|
||||
::run(row, col, m_lhs, m_rhs, res);
|
||||
ScalarCoeffImpl::run(row, col, m_lhs, m_rhs, res);
|
||||
return res;
|
||||
}
|
||||
|
||||
template<int LoadMode>
|
||||
const PacketScalar _packet(int row, int col) const
|
||||
{
|
||||
const bool unroll = CoeffReadCost <= EIGEN_UNROLLING_LIMIT;
|
||||
PacketScalar res;
|
||||
ei_packet_product_impl<Flags&RowMajorBit ? true : false, Lhs::ColsAtCompileTime-1,
|
||||
unroll ? Lhs::ColsAtCompileTime : Dynamic,
|
||||
_LhsNested, _RhsNested, PacketScalar>
|
||||
::run(row, col, m_lhs, m_rhs, res);
|
||||
PacketCoeffImpl::run(row, col, m_lhs, m_rhs, res);
|
||||
return res;
|
||||
}
|
||||
|
||||
@@ -302,11 +243,11 @@ template<typename Lhs, typename Rhs, int EvalMode> class Product : ei_no_assignm
|
||||
*/
|
||||
template<typename Derived>
|
||||
template<typename OtherDerived>
|
||||
inline const typename MatrixBase<Derived>::template ProductReturnType<OtherDerived>::Type
|
||||
inline const typename ProductReturnType<Derived,OtherDerived>::Type
|
||||
MatrixBase<Derived>::operator*(const MatrixBase<OtherDerived> &other) const
|
||||
{
|
||||
assert( (Derived::Flags&ArrayBit) == (OtherDerived::Flags&ArrayBit) );
|
||||
return typename ProductReturnType<OtherDerived>::Type(derived(), other.derived());
|
||||
return typename ProductReturnType<Derived,OtherDerived>::Type(derived(), other.derived());
|
||||
}
|
||||
|
||||
/** replaces \c *this by \c *this * \a other.
|
||||
@@ -321,6 +262,157 @@ MatrixBase<Derived>::operator*=(const MatrixBase<OtherDerived> &other)
|
||||
return *this = *this * other;
|
||||
}
|
||||
|
||||
/***************************************************************************
|
||||
* Normal product .coeff() implementation (with meta-unrolling)
|
||||
***************************************************************************/
|
||||
|
||||
/**************************************
|
||||
*** Scalar path - no vectorization ***
|
||||
**************************************/
|
||||
|
||||
template<int Index, typename Lhs, typename Rhs>
|
||||
struct ei_product_coeff_impl<NoVectorization, Index, Lhs, Rhs>
|
||||
{
|
||||
inline static void run(int row, int col, const Lhs& lhs, const Rhs& rhs, typename Lhs::Scalar &res)
|
||||
{
|
||||
ei_product_coeff_impl<NoVectorization, Index-1, Lhs, Rhs>::run(row, col, lhs, rhs, res);
|
||||
res += lhs.coeff(row, Index) * rhs.coeff(Index, col);
|
||||
}
|
||||
};
|
||||
|
||||
template<typename Lhs, typename Rhs>
|
||||
struct ei_product_coeff_impl<NoVectorization, 0, Lhs, Rhs>
|
||||
{
|
||||
inline static void run(int row, int col, const Lhs& lhs, const Rhs& rhs, typename Lhs::Scalar &res)
|
||||
{
|
||||
res = lhs.coeff(row, 0) * rhs.coeff(0, col);
|
||||
}
|
||||
};
|
||||
|
||||
template<typename Lhs, typename Rhs>
|
||||
struct ei_product_coeff_impl<NoVectorization, Dynamic, Lhs, Rhs>
|
||||
{
|
||||
inline static void run(int row, int col, const Lhs& lhs, const Rhs& rhs, typename Lhs::Scalar& res)
|
||||
{
|
||||
res = lhs.coeff(row, 0) * rhs.coeff(0, col);
|
||||
for(int i = 1; i < lhs.cols(); i++)
|
||||
res += lhs.coeff(row, i) * rhs.coeff(i, col);
|
||||
}
|
||||
};
|
||||
|
||||
// prevent buggy user code from causing an infinite recursion
|
||||
template<typename Lhs, typename Rhs>
|
||||
struct ei_product_coeff_impl<NoVectorization, -1, Lhs, Rhs>
|
||||
{
|
||||
inline static void run(int, int, const Lhs&, const Rhs&, typename Lhs::Scalar&) {}
|
||||
};
|
||||
|
||||
/*******************************************
|
||||
*** Scalar path with inner vectorization ***
|
||||
*******************************************/
|
||||
|
||||
template<int Index, typename Lhs, typename Rhs, typename PacketScalar>
|
||||
struct ei_product_coeff_vectorized_impl
|
||||
{
|
||||
enum { PacketSize = ei_packet_traits<typename Lhs::Scalar>::size };
|
||||
inline static void run(int row, int col, const Lhs& lhs, const Rhs& rhs, typename Lhs::PacketScalar &pres)
|
||||
{
|
||||
ei_product_coeff_vectorized_impl<Index-PacketSize, Lhs, Rhs, PacketScalar>::run(row, col, lhs, rhs, pres);
|
||||
pres = ei_padd(pres, ei_pmul( lhs.template packet<Aligned>(row, Index) , rhs.template packet<Aligned>(Index, col) ));
|
||||
}
|
||||
};
|
||||
|
||||
template<typename Lhs, typename Rhs, typename PacketScalar>
|
||||
struct ei_product_coeff_vectorized_impl<0, Lhs, Rhs, PacketScalar>
|
||||
{
|
||||
inline static void run(int row, int col, const Lhs& lhs, const Rhs& rhs, typename Lhs::PacketScalar &pres)
|
||||
{
|
||||
pres = ei_pmul(lhs.template packet<Aligned>(row, 0) , rhs.template packet<Aligned>(0, col));
|
||||
}
|
||||
};
|
||||
|
||||
template<int Index, typename Lhs, typename Rhs>
|
||||
struct ei_product_coeff_impl<InnerVectorization, Index, Lhs, Rhs>
|
||||
{
|
||||
typedef typename Lhs::PacketScalar PacketScalar;
|
||||
enum { PacketSize = ei_packet_traits<typename Lhs::Scalar>::size };
|
||||
inline static void run(int row, int col, const Lhs& lhs, const Rhs& rhs, typename Lhs::Scalar &res)
|
||||
{
|
||||
PacketScalar pres;
|
||||
ei_product_coeff_vectorized_impl<Index+1-PacketSize, Lhs, Rhs, PacketScalar>::run(row, col, lhs, rhs, pres);
|
||||
ei_product_coeff_impl<NoVectorization,Index,Lhs,Rhs>::run(row, col, lhs, rhs, res);
|
||||
res = ei_predux(pres);
|
||||
}
|
||||
};
|
||||
|
||||
/*******************
|
||||
*** Packet path ***
|
||||
*******************/
|
||||
|
||||
template<int Index, typename Lhs, typename Rhs, typename PacketScalar>
|
||||
struct ei_product_packet_impl<RowMajorProduct, Index, Lhs, Rhs, PacketScalar>
|
||||
{
|
||||
inline static void run(int row, int col, const Lhs& lhs, const Rhs& rhs, PacketScalar &res)
|
||||
{
|
||||
ei_product_packet_impl<RowMajorProduct, Index-1, Lhs, Rhs, PacketScalar>::run(row, col, lhs, rhs, res);
|
||||
res = ei_pmadd(ei_pset1(lhs.coeff(row, Index)), rhs.template packet<Aligned>(Index, col), res);
|
||||
}
|
||||
};
|
||||
|
||||
template<int Index, typename Lhs, typename Rhs, typename PacketScalar>
|
||||
struct ei_product_packet_impl<ColMajorProduct, Index, Lhs, Rhs, PacketScalar>
|
||||
{
|
||||
inline static void run(int row, int col, const Lhs& lhs, const Rhs& rhs, PacketScalar &res)
|
||||
{
|
||||
ei_product_packet_impl<ColMajorProduct, Index-1, Lhs, Rhs, PacketScalar>::run(row, col, lhs, rhs, res);
|
||||
res = ei_pmadd(lhs.template packet<Aligned>(row, Index), ei_pset1(rhs.coeff(Index, col)), res);
|
||||
}
|
||||
};
|
||||
|
||||
template<typename Lhs, typename Rhs, typename PacketScalar>
|
||||
struct ei_product_packet_impl<RowMajorProduct, 0, Lhs, Rhs, PacketScalar>
|
||||
{
|
||||
inline static void run(int row, int col, const Lhs& lhs, const Rhs& rhs, PacketScalar &res)
|
||||
{
|
||||
res = ei_pmul(ei_pset1(lhs.coeff(row, 0)),rhs.template packet<Aligned>(0, col));
|
||||
}
|
||||
};
|
||||
|
||||
template<typename Lhs, typename Rhs, typename PacketScalar>
|
||||
struct ei_product_packet_impl<ColMajorProduct, 0, Lhs, Rhs, PacketScalar>
|
||||
{
|
||||
inline static void run(int row, int col, const Lhs& lhs, const Rhs& rhs, PacketScalar &res)
|
||||
{
|
||||
res = ei_pmul(lhs.template packet<Aligned>(row, 0), ei_pset1(rhs.coeff(0, col)));
|
||||
}
|
||||
};
|
||||
|
||||
template<int StorageOrder, typename Lhs, typename Rhs, typename PacketScalar>
|
||||
struct ei_product_packet_impl<StorageOrder, Dynamic, Lhs, Rhs, PacketScalar>
|
||||
{
|
||||
inline static void run(int row, int col, const Lhs& lhs, const Rhs& rhs, PacketScalar& res)
|
||||
{
|
||||
res = ei_pmul(ei_pset1(lhs.coeff(row, 0)),rhs.template packet<Aligned>(0, col));
|
||||
for(int i = 1; i < lhs.cols(); i++)
|
||||
res = ei_pmadd(ei_pset1(lhs.coeff(row, i)), rhs.template packet<Aligned>(i, col), res);
|
||||
}
|
||||
};
|
||||
|
||||
template<typename Lhs, typename Rhs, typename PacketScalar>
|
||||
struct ei_product_packet_impl<ColMajorProduct, Dynamic, Lhs, Rhs, PacketScalar>
|
||||
{
|
||||
inline static void run(int row, int col, const Lhs& lhs, const Rhs& rhs, PacketScalar& res)
|
||||
{
|
||||
res = ei_pmul(lhs.template packet<Aligned>(row, 0), ei_pset1(rhs.coeff(0, col)));
|
||||
for(int i = 1; i < lhs.cols(); i++)
|
||||
res = ei_pmadd(lhs.template packet<Aligned>(row, i), ei_pset1(rhs.coeff(i, col)), res);
|
||||
}
|
||||
};
|
||||
|
||||
/***************************************************************************
|
||||
* Cache friendly product callers and specific nested evaluation strategies
|
||||
***************************************************************************/
|
||||
|
||||
/** \internal */
|
||||
template<typename Derived>
|
||||
template<typename Lhs,typename Rhs>
|
||||
@@ -339,6 +431,28 @@ inline Derived& MatrixBase<Derived>::lazyAssign(const Product<Lhs,Rhs,CacheFrien
|
||||
return derived();
|
||||
}
|
||||
|
||||
template<typename T> class ei_product_eval_to_column_major
|
||||
{
|
||||
typedef typename ei_traits<T>::Scalar _Scalar;
|
||||
enum {
|
||||
_Rows = ei_traits<T>::RowsAtCompileTime,
|
||||
_Cols = ei_traits<T>::ColsAtCompileTime,
|
||||
_MaxRows = ei_traits<T>::MaxRowsAtCompileTime,
|
||||
_MaxCols = ei_traits<T>::MaxColsAtCompileTime,
|
||||
_Flags = ei_traits<T>::Flags
|
||||
};
|
||||
|
||||
public:
|
||||
typedef Matrix<_Scalar,
|
||||
_Rows, _Cols, _MaxRows, _MaxCols,
|
||||
ei_corrected_matrix_flags<
|
||||
_Scalar,
|
||||
_Rows, _Cols, _MaxRows, _MaxCols,
|
||||
_Flags
|
||||
>::ret & ~RowMajorBit
|
||||
> type;
|
||||
};
|
||||
|
||||
template<typename T> struct ei_product_copy_rhs
|
||||
{
|
||||
typedef typename ei_meta_if<
|
||||
|
||||
Reference in New Issue
Block a user