devtools/eigen
1
0
Fork 0
mirror of https://gitlab.com/libeigen/eigen.git synced 2026-04-10 11:34:33 +08:00
Code Issues Packages Projects Releases Wiki Activity

* Started support for unaligned vectorization.

Browse Source 
* Introduce a new highly optimized matrix-matrix product for large
  matrices. The code is still highly experimental and it is activated
  only if you define EIGEN_WIP_PRODUCT at compile time.
  Currently the third dimension of the product must be a factor of
  the packet size (x4 for floats) and the right handed side matrix
  must be column major.
  Moreover, currently c = a*b; actually computes c += a*b !!
  Therefore, the code is provided for experimentation purpose only !
  These limitations will be fixed soon or later to become the default
  product implementation.
This commit is contained in:
Gael Guennebaud
2008-05-05 10:23:29 +00:00
parent 8c6007f80e
commit 46fa4c713f
15 changed files with 663 additions and 79 deletions
Download Patch File Download Diff File Expand all files Collapse all files

89
Eigen/src/Core/Product.h
View File

@@ -69,7 +69,7 @@ struct ei_packet_product_unroller<true, Index, Size, Lhs, Rhs, PacketScalar>
static void run(int row, int col, const Lhs& lhs, const Rhs& rhs, PacketScalar &res)
{
ei_packet_product_unroller<true, Index-1, Size, Lhs, Rhs, PacketScalar>::run(row, col, lhs, rhs, res);
res = ei_pmadd(ei_pset1(lhs.coeff(row, Index)), rhs.packetCoeff(Index, col), res);
res = ei_pmadd(ei_pset1(lhs.coeff(row, Index)), rhs.template packetCoeff<Aligned>(Index, col), res);
}
};
@@ -79,7 +79,7 @@ struct ei_packet_product_unroller<false, Index, Size, Lhs, Rhs, PacketScalar>
static void run(int row, int col, const Lhs& lhs, const Rhs& rhs, PacketScalar &res)
{
ei_packet_product_unroller<false, Index-1, Size, Lhs, Rhs, PacketScalar>::run(row, col, lhs, rhs, res);
res = ei_pmadd(lhs.packetCoeff(row, Index), ei_pset1(rhs.coeff(Index, col)), res);
res = ei_pmadd(lhs.template packetCoeff<Aligned>(row, Index), ei_pset1(rhs.coeff(Index, col)), res);
}
};
@@ -88,7 +88,7 @@ struct ei_packet_product_unroller<true, 0, Size, Lhs, Rhs, PacketScalar>
{
static void run(int row, int col, const Lhs& lhs, const Rhs& rhs, PacketScalar &res)
{
res = ei_pmul(ei_pset1(lhs.coeff(row, 0)),rhs.packetCoeff(0, col));
res = ei_pmul(ei_pset1(lhs.coeff(row, 0)),rhs.template packetCoeff<Aligned>(0, col));
}
};
@@ -97,7 +97,7 @@ struct ei_packet_product_unroller<false, 0, Size, Lhs, Rhs, PacketScalar>
{
static void run(int row, int col, const Lhs& lhs, const Rhs& rhs, PacketScalar &res)
{
res = ei_pmul(lhs.packetCoeff(row, 0), ei_pset1(rhs.coeff(0, col)));
res = ei_pmul(lhs.template packetCoeff<Aligned>(row, 0), ei_pset1(rhs.coeff(0, col)));
}
};
@@ -196,10 +196,10 @@ template<typename Lhs, typename Rhs, int EvalMode> class Product : ei_no_assignm
}
/** \internal */
template<typename DestDerived>
template<typename DestDerived, int AlignedMode>
void _cacheOptimalEval(DestDerived& res, ei_meta_false) const;
#ifdef EIGEN_VECTORIZE
template<typename DestDerived>
template<typename DestDerived, int AlignedMode>
void _cacheOptimalEval(DestDerived& res, ei_meta_true) const;
#endif
@@ -228,6 +228,7 @@ template<typename Lhs, typename Rhs, int EvalMode> class Product : ei_no_assignm
return res;
}
template<int LoadMode>
PacketScalar _packetCoeff(int row, int col) const
{
if(Lhs::ColsAtCompileTime <= EIGEN_UNROLLING_LIMIT)
@@ -247,21 +248,30 @@ template<typename Lhs, typename Rhs, int EvalMode> class Product : ei_no_assignm
PacketScalar _packetCoeffRowMajor(int row, int col) const
{
PacketScalar res;
res = ei_pmul(ei_pset1(m_lhs.coeff(row, 0)),m_rhs.packetCoeff(0, col));
res = ei_pmul(ei_pset1(m_lhs.coeff(row, 0)),m_rhs.template packetCoeff<Aligned>(0, col));
for(int i = 1; i < m_lhs.cols(); i++)
res = ei_pmadd(ei_pset1(m_lhs.coeff(row, i)), m_rhs.packetCoeff(i, col), res);
res = ei_pmadd(ei_pset1(m_lhs.coeff(row, i)), m_rhs.template packetCoeff<Aligned>(i, col), res);
return res;
}
PacketScalar _packetCoeffColumnMajor(int row, int col) const
{
PacketScalar res;
res = ei_pmul(m_lhs.packetCoeff(row, 0), ei_pset1(m_rhs.coeff(0, col)));
res = ei_pmul(m_lhs.template packetCoeff<Aligned>(row, 0), ei_pset1(m_rhs.coeff(0, col)));
for(int i = 1; i < m_lhs.cols(); i++)
res = ei_pmadd(m_lhs.packetCoeff(row, i), ei_pset1(m_rhs.coeff(i, col)), res);
res = ei_pmadd(m_lhs.template packetCoeff<Aligned>(row, i), ei_pset1(m_rhs.coeff(i, col)), res);
return res;
// const PacketScalar tmp[4];
// ei_punpack(m_rhs.packetCoeff(0,col), tmp);
//
// return
// ei_pmadd(m_lhs.packetCoeff(row, 0), tmp[0],
// ei_pmadd(m_lhs.packetCoeff(row, 1), tmp[1],
// ei_pmadd(m_lhs.packetCoeff(row, 2), tmp[2]
// ei_pmul(m_lhs.packetCoeff(row, 3), tmp[3]))));
}
protected:
const LhsNested m_lhs;
const RhsNested m_rhs;
@@ -298,7 +308,7 @@ template<typename Derived>
template<typename Lhs, typename Rhs>
Derived& MatrixBase<Derived>::lazyAssign(const Product<Lhs,Rhs,CacheOptimalProduct>& product)
{
product._cacheOptimalEval(*this,
product.template _cacheOptimalEval<Derived, Aligned>(derived(),
#ifdef EIGEN_VECTORIZE
typename ei_meta_if<Flags & VectorizableBit, ei_meta_true, ei_meta_false>::ret()
#else
@@ -309,7 +319,7 @@ Derived& MatrixBase<Derived>::lazyAssign(const Product<Lhs,Rhs,CacheOptimalProdu
}
template<typename Lhs, typename Rhs, int EvalMode>
template<typename DestDerived>
template<typename DestDerived, int AlignedMode>
void Product<Lhs,Rhs,EvalMode>::_cacheOptimalEval(DestDerived& res, ei_meta_false) const
{
res.setZero();
@@ -372,14 +382,14 @@ void Product<Lhs,Rhs,EvalMode>::_cacheOptimalEval(DestDerived& res, ei_meta_fals
#ifdef EIGEN_VECTORIZE
template<typename Lhs, typename Rhs, int EvalMode>
template<typename DestDerived>
template<typename DestDerived, int AlignedMode>
void Product<Lhs,Rhs,EvalMode>::_cacheOptimalEval(DestDerived& res, ei_meta_true) const
{
if (((Lhs::Flags&RowMajorBit) && (_cols() % ei_packet_traits<Scalar>::size != 0))
|| (_rows() % ei_packet_traits<Scalar>::size != 0))
{
return _cacheOptimalEval(res, ei_meta_false());
return _cacheOptimalEval<DestDerived, AlignedMode>(res, ei_meta_false());
}
res.setZero();
@@ -398,12 +408,12 @@ void Product<Lhs,Rhs,EvalMode>::_cacheOptimalEval(DestDerived& res, ei_meta_true
const typename ei_packet_traits<Scalar>::type tmp3 = ei_pset1(m_lhs.coeff(k,j+3));
for (int i=0; i<this->cols(); i+=ei_packet_traits<Scalar>::size)
{
res.writePacketCoeff(k,i,
ei_pmadd(tmp0, m_rhs.packetCoeff(j+0,i),
ei_pmadd(tmp1, m_rhs.packetCoeff(j+1,i),
ei_pmadd(tmp2, m_rhs.packetCoeff(j+2,i),
ei_pmadd(tmp3, m_rhs.packetCoeff(j+3,i),
res.packetCoeff(k,i)))))
res.template writePacketCoeff<AlignedMode>(k,i,
ei_pmadd(tmp0, m_rhs.template packetCoeff<AlignedMode>(j+0,i),
ei_pmadd(tmp1, m_rhs.template packetCoeff<AlignedMode>(j+1,i),
ei_pmadd(tmp2, m_rhs.template packetCoeff<AlignedMode>(j+2,i),
ei_pmadd(tmp3, m_rhs.template packetCoeff<AlignedMode>(j+3,i),
res.template packetCoeff<AlignedMode>(k,i)))))
);
}
}
@@ -414,41 +424,44 @@ void Product<Lhs,Rhs,EvalMode>::_cacheOptimalEval(DestDerived& res, ei_meta_true
{
const typename ei_packet_traits<Scalar>::type tmp = ei_pset1(m_lhs.coeff(k,j));
for (int i=0; i<this->cols(); i+=ei_packet_traits<Scalar>::size)
res.writePacketCoeff(k,i, ei_pmadd(tmp, m_rhs.packetCoeff(j,i), res.packetCoeff(k,i)));
res.template writePacketCoeff<AlignedMode>(k,i,
ei_pmadd(tmp, m_rhs.template packetCoeff<AlignedMode>(j,i), res.template packetCoeff<AlignedMode>(k,i)));
}
}
}
else
{
// std::cout << "packet lhs\n";
int j=0;
for(; j<cols4; j+=4)
int k=0;
for(; k<cols4; k+=4)
{
for(int k=0; k<this->cols(); k++)
for(int j=0; j<this->cols(); j+=1)
{
const typename ei_packet_traits<Scalar>::type tmp0 = ei_pset1(m_rhs.coeff(j+0,k));
const typename ei_packet_traits<Scalar>::type tmp1 = ei_pset1(m_rhs.coeff(j+1,k));
const typename ei_packet_traits<Scalar>::type tmp2 = ei_pset1(m_rhs.coeff(j+2,k));
const typename ei_packet_traits<Scalar>::type tmp3 = ei_pset1(m_rhs.coeff(j+3,k));
const typename ei_packet_traits<Scalar>::type tmp0 = ei_pset1(m_rhs.coeff(k+0,j));
const typename ei_packet_traits<Scalar>::type tmp1 = ei_pset1(m_rhs.coeff(k+1,j));
const typename ei_packet_traits<Scalar>::type tmp2 = ei_pset1(m_rhs.coeff(k+2,j));
const typename ei_packet_traits<Scalar>::type tmp3 = ei_pset1(m_rhs.coeff(k+3,j));
for (int i=0; i<this->rows(); i+=ei_packet_traits<Scalar>::size)
{
res.writePacketCoeff(i,k,
ei_pmadd(tmp0, m_lhs.packetCoeff(i,j),
ei_pmadd(tmp1, m_lhs.packetCoeff(i,j+1),
ei_pmadd(tmp2, m_lhs.packetCoeff(i,j+2),
ei_pmadd(tmp3, m_lhs.packetCoeff(i,j+3),
res.packetCoeff(i,k)))))
res.template writePacketCoeff<AlignedMode>(i,j,
ei_pmadd(tmp0, m_lhs.template packetCoeff<AlignedMode>(i,k),
ei_pmadd(tmp1, m_lhs.template packetCoeff<AlignedMode>(i,k+1),
ei_pmadd(tmp2, m_lhs.template packetCoeff<AlignedMode>(i,k+2),
ei_pmadd(tmp3, m_lhs.template packetCoeff<AlignedMode>(i,k+3),
res.template packetCoeff<AlignedMode>(i,j)))))
);
}
}
}
for(; j<m_lhs.cols(); ++j)
for(; k<m_lhs.cols(); ++k)
{
for(int k=0; k<this->cols(); k++)
for(int j=0; j<this->cols(); j++)
{
const typename ei_packet_traits<Scalar>::type tmp = ei_pset1(m_rhs.coeff(j,k));
const typename ei_packet_traits<Scalar>::type tmp = ei_pset1(m_rhs.coeff(k,j));
for (int i=0; i<this->rows(); i+=ei_packet_traits<Scalar>::size)
res.writePacketCoeff(i,k, ei_pmadd(tmp, m_lhs.packetCoeff(i,j), res.packetCoeff(i,k)));
res.template writePacketCoeff<AlignedMode>(k,j,
ei_pmadd(tmp, m_lhs.template packetCoeff<AlignedMode>(i,k), res.template packetCoeff<AlignedMode>(i,j)));
}
}
}