devtools/eigen
1
0
Fork 0
mirror of https://gitlab.com/libeigen/eigen.git synced 2026-04-10 11:34:33 +08:00
Code Issues Packages Projects Releases Wiki Activity

Apply clang-format

Browse Source
This commit is contained in:
Tobias Wood
2023-11-29 11:12:48 +00:00
parent 9ea520fc45
commit f38e16c193
534 changed files with 103368 additions and 116934 deletions
Download Patch File Download Diff File Expand all files Collapse all files

327
unsupported/Eigen/CXX11/src/Tensor/TensorPadding.h
View File

@@ -16,16 +16,15 @@
namespace Eigen {
/** \class TensorPadding
* \ingroup CXX11_Tensor_Module
*
* \brief Tensor padding class.
* At the moment only padding with a constant value is supported.
*
*/
* \ingroup CXX11_Tensor_Module
*
* \brief Tensor padding class.
* At the moment only padding with a constant value is supported.
*
*/
namespace internal {
template<typename PaddingDimensions, typename XprType>
struct traits<TensorPaddingOp<PaddingDimensions, XprType> > : public traits<XprType>
{
template <typename PaddingDimensions, typename XprType>
struct traits<TensorPaddingOp<PaddingDimensions, XprType> > : public traits<XprType> {
typedef typename XprType::Scalar Scalar;
typedef traits<XprType> XprTraits;
typedef typename XprTraits::StorageKind StorageKind;
@@ -37,26 +36,22 @@ struct traits<TensorPaddingOp<PaddingDimensions, XprType> > : public traits<XprT
typedef typename XprTraits::PointerType PointerType;
};
template<typename PaddingDimensions, typename XprType>
struct eval<TensorPaddingOp<PaddingDimensions, XprType>, Eigen::Dense>
{
template <typename PaddingDimensions, typename XprType>
struct eval<TensorPaddingOp<PaddingDimensions, XprType>, Eigen::Dense> {
typedef const TensorPaddingOp<PaddingDimensions, XprType>& type;
};
template<typename PaddingDimensions, typename XprType>
struct nested<TensorPaddingOp<PaddingDimensions, XprType>, 1, typename eval<TensorPaddingOp<PaddingDimensions, XprType> >::type>
{
template <typename PaddingDimensions, typename XprType>
struct nested<TensorPaddingOp<PaddingDimensions, XprType>, 1,
typename eval<TensorPaddingOp<PaddingDimensions, XprType> >::type> {
typedef TensorPaddingOp<PaddingDimensions, XprType> type;
};
} // end namespace internal
template<typename PaddingDimensions, typename XprType>
class TensorPaddingOp : public TensorBase<TensorPaddingOp<PaddingDimensions, XprType>, ReadOnlyAccessors>
{
public:
template <typename PaddingDimensions, typename XprType>
class TensorPaddingOp : public TensorBase<TensorPaddingOp<PaddingDimensions, XprType>, ReadOnlyAccessors> {
public:
typedef typename Eigen::internal::traits<TensorPaddingOp>::Scalar Scalar;
typedef typename Eigen::NumTraits<Scalar>::Real RealScalar;
typedef typename XprType::CoeffReturnType CoeffReturnType;
@@ -64,29 +59,24 @@ class TensorPaddingOp : public TensorBase<TensorPaddingOp<PaddingDimensions, Xpr
typedef typename Eigen::internal::traits<TensorPaddingOp>::StorageKind StorageKind;
typedef typename Eigen::internal::traits<TensorPaddingOp>::Index Index;
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorPaddingOp(const XprType& expr, const PaddingDimensions& padding_dims, const Scalar padding_value)
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorPaddingOp(const XprType& expr, const PaddingDimensions& padding_dims,
const Scalar padding_value)
: m_xpr(expr), m_padding_dims(padding_dims), m_padding_value(padding_value) {}
EIGEN_DEVICE_FUNC
const PaddingDimensions& padding() const { return m_padding_dims; }
EIGEN_DEVICE_FUNC
Scalar padding_value() const { return m_padding_value; }
EIGEN_DEVICE_FUNC const PaddingDimensions& padding() const { return m_padding_dims; }
EIGEN_DEVICE_FUNC Scalar padding_value() const { return m_padding_value; }
EIGEN_DEVICE_FUNC
const internal::remove_all_t<typename XprType::Nested>&
expression() const { return m_xpr; }
EIGEN_DEVICE_FUNC const internal::remove_all_t<typename XprType::Nested>& expression() const { return m_xpr; }
protected:
typename XprType::Nested m_xpr;
const PaddingDimensions m_padding_dims;
const Scalar m_padding_value;
protected:
typename XprType::Nested m_xpr;
const PaddingDimensions m_padding_dims;
const Scalar m_padding_value;
};
// Eval as rvalue
template<typename PaddingDimensions, typename ArgType, typename Device>
struct TensorEvaluator<const TensorPaddingOp<PaddingDimensions, ArgType>, Device>
{
template <typename PaddingDimensions, typename ArgType, typename Device>
struct TensorEvaluator<const TensorPaddingOp<PaddingDimensions, ArgType>, Device> {
typedef TensorPaddingOp<PaddingDimensions, ArgType> XprType;
typedef typename XprType::Index Index;
static constexpr int NumDims = internal::array_size<PaddingDimensions>::value;
@@ -100,12 +90,12 @@ struct TensorEvaluator<const TensorPaddingOp<PaddingDimensions, ArgType>, Device
static constexpr int Layout = TensorEvaluator<ArgType, Device>::Layout;
enum {
IsAligned = true,
PacketAccess = TensorEvaluator<ArgType, Device>::PacketAccess,
BlockAccess = TensorEvaluator<ArgType, Device>::RawAccess,
IsAligned = true,
PacketAccess = TensorEvaluator<ArgType, Device>::PacketAccess,
BlockAccess = TensorEvaluator<ArgType, Device>::RawAccess,
PreferBlockAccess = true,
CoordAccess = true,
RawAccess = false
CoordAccess = true,
RawAccess = false
};
typedef std::remove_const_t<Scalar> ScalarNoConst;
@@ -114,14 +104,11 @@ struct TensorEvaluator<const TensorPaddingOp<PaddingDimensions, ArgType>, Device
typedef internal::TensorBlockDescriptor<NumDims, Index> TensorBlockDesc;
typedef internal::TensorBlockScratchAllocator<Device> TensorBlockScratch;
typedef typename internal::TensorMaterializedBlock<ScalarNoConst, NumDims,
Layout, Index>
TensorBlock;
typedef typename internal::TensorMaterializedBlock<ScalarNoConst, NumDims, Layout, Index> TensorBlock;
//===--------------------------------------------------------------------===//
EIGEN_STRONG_INLINE TensorEvaluator(const XprType& op, const Device& device)
: m_impl(op.expression(), device), m_padding(op.padding()), m_paddingValue(op.padding_value()), m_device(device)
{
: m_impl(op.expression(), device), m_padding(op.padding()), m_paddingValue(op.padding_value()), m_device(device) {
// The padding op doesn't change the rank of the tensor. Directly padding a scalar would lead
// to a vector, which doesn't make sense. Instead one should reshape the scalar into a vector
// of 1 element first and then pad.
@@ -137,16 +124,16 @@ struct TensorEvaluator<const TensorPaddingOp<PaddingDimensions, ArgType>, Device
m_inputStrides[0] = 1;
m_outputStrides[0] = 1;
for (int i = 1; i < NumDims; ++i) {
m_inputStrides[i] = m_inputStrides[i-1] * input_dims[i-1];
m_outputStrides[i] = m_outputStrides[i-1] * m_dimensions[i-1];
m_inputStrides[i] = m_inputStrides[i - 1] * input_dims[i - 1];
m_outputStrides[i] = m_outputStrides[i - 1] * m_dimensions[i - 1];
}
m_outputStrides[NumDims] = m_outputStrides[NumDims-1] * m_dimensions[NumDims-1];
m_outputStrides[NumDims] = m_outputStrides[NumDims - 1] * m_dimensions[NumDims - 1];
} else {
m_inputStrides[NumDims - 1] = 1;
m_outputStrides[NumDims] = 1;
for (int i = NumDims - 2; i >= 0; --i) {
m_inputStrides[i] = m_inputStrides[i+1] * input_dims[i+1];
m_outputStrides[i+1] = m_outputStrides[i+2] * m_dimensions[i+1];
m_inputStrides[i] = m_inputStrides[i + 1] * input_dims[i + 1];
m_outputStrides[i + 1] = m_outputStrides[i + 2] * m_dimensions[i + 1];
}
m_outputStrides[0] = m_outputStrides[1] * m_dimensions[0];
}
@@ -161,18 +148,14 @@ struct TensorEvaluator<const TensorPaddingOp<PaddingDimensions, ArgType>, Device
#ifdef EIGEN_USE_THREADS
template <typename EvalSubExprsCallback>
EIGEN_STRONG_INLINE void evalSubExprsIfNeededAsync(
EvaluatorPointerType, EvalSubExprsCallback done) {
EIGEN_STRONG_INLINE void evalSubExprsIfNeededAsync(EvaluatorPointerType, EvalSubExprsCallback done) {
m_impl.evalSubExprsIfNeededAsync(nullptr, [done](bool) { done(true); });
}
#endif // EIGEN_USE_THREADS
EIGEN_STRONG_INLINE void cleanup() {
m_impl.cleanup();
}
EIGEN_STRONG_INLINE void cleanup() { m_impl.cleanup(); }
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType coeff(Index index) const
{
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType coeff(Index index) const {
eigen_assert(index < dimensions().TotalSize());
Index inputIndex = 0;
if (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
@@ -192,24 +175,23 @@ struct TensorEvaluator<const TensorPaddingOp<PaddingDimensions, ArgType>, Device
} else {
EIGEN_UNROLL_LOOP
for (int i = 0; i < NumDims - 1; ++i) {
const Index idx = index / m_outputStrides[i+1];
const Index idx = index / m_outputStrides[i + 1];
if (isPaddingAtIndexForDim(idx, i)) {
return m_paddingValue;
}
inputIndex += (idx - m_padding[i].first) * m_inputStrides[i];
index -= idx * m_outputStrides[i+1];
index -= idx * m_outputStrides[i + 1];
}
if (isPaddingAtIndexForDim(index, NumDims-1)) {
if (isPaddingAtIndexForDim(index, NumDims - 1)) {
return m_paddingValue;
}
inputIndex += (index - m_padding[NumDims-1].first);
inputIndex += (index - m_padding[NumDims - 1].first);
}
return m_impl.coeff(inputIndex);
}
template<int LoadMode>
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketReturnType packet(Index index) const
{
template <int LoadMode>
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketReturnType packet(Index index) const {
if (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
return packetColMajor(index);
}
@@ -220,31 +202,25 @@ struct TensorEvaluator<const TensorPaddingOp<PaddingDimensions, ArgType>, Device
TensorOpCost cost = m_impl.costPerCoeff(vectorized);
if (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
EIGEN_UNROLL_LOOP
for (int i = 0; i < NumDims; ++i)
updateCostPerDimension(cost, i, i == 0);
for (int i = 0; i < NumDims; ++i) updateCostPerDimension(cost, i, i == 0);
} else {
EIGEN_UNROLL_LOOP
for (int i = NumDims - 1; i >= 0; --i)
updateCostPerDimension(cost, i, i == NumDims - 1);
for (int i = NumDims - 1; i >= 0; --i) updateCostPerDimension(cost, i, i == NumDims - 1);
}
return cost;
}
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
internal::TensorBlockResourceRequirements getResourceRequirements() const {
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE internal::TensorBlockResourceRequirements getResourceRequirements() const {
const size_t target_size = m_device.lastLevelCacheSize();
return internal::TensorBlockResourceRequirements::merge(
internal::TensorBlockResourceRequirements::skewed<Scalar>(target_size),
m_impl.getResourceRequirements());
internal::TensorBlockResourceRequirements::skewed<Scalar>(target_size), m_impl.getResourceRequirements());
}
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorBlock
block(TensorBlockDesc& desc, TensorBlockScratch& scratch,
bool /*root_of_expr_ast*/ = false) const {
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorBlock block(TensorBlockDesc& desc, TensorBlockScratch& scratch,
bool /*root_of_expr_ast*/ = false) const {
// If one of the dimensions is zero, return empty block view.
if (desc.size() == 0) {
return TensorBlock(internal::TensorBlockKind::kView, NULL,
desc.dimensions());
return TensorBlock(internal::TensorBlockKind::kView, NULL, desc.dimensions());
}
static const bool IsColMajor = Layout == static_cast<int>(ColMajor);
@@ -282,8 +258,7 @@ struct TensorEvaluator<const TensorPaddingOp<PaddingDimensions, ArgType>, Device
// same dimensions as the requested block (for destination buffer this
// property is guaranteed by `desc.destination()`).
Index output_offset = 0;
const DSizes<Index, NumDims> output_strides =
internal::strides<Layout>(desc.dimensions());
const DSizes<Index, NumDims> output_strides = internal::strides<Layout>(desc.dimensions());
// NOTE(ezhulenev): We initialize bock iteration state for `NumDims - 1`
// dimensions, skipping innermost dimension. In theory it should be possible
@@ -306,8 +281,7 @@ struct TensorEvaluator<const TensorPaddingOp<PaddingDimensions, ArgType>, Device
it[i].output_span = it[i].output_stride * (it[i].size - 1);
}
const Index input_inner_dim_size =
static_cast<Index>(m_impl.dimensions()[inner_dim_idx]);
const Index input_inner_dim_size = static_cast<Index>(m_impl.dimensions()[inner_dim_idx]);
// Total output size.
const Index output_size = desc.size();
@@ -321,8 +295,7 @@ struct TensorEvaluator<const TensorPaddingOp<PaddingDimensions, ArgType>, Device
// dimension.
const Index output_inner_pad_before_size =
input_offsets[inner_dim_idx] < 0
? numext::mini(numext::abs(input_offsets[inner_dim_idx]),
output_inner_dim_size)
? numext::mini(numext::abs(input_offsets[inner_dim_idx]), output_inner_dim_size)
: 0;
// How many values we can actually copy from the input inner dimension.
@@ -330,22 +303,18 @@ struct TensorEvaluator<const TensorPaddingOp<PaddingDimensions, ArgType>, Device
// Want to copy from input.
(output_inner_dim_size - output_inner_pad_before_size),
// Can copy from input.
numext::maxi(input_inner_dim_size - (input_offsets[inner_dim_idx] +
output_inner_pad_before_size),
Index(0)));
numext::maxi(input_inner_dim_size - (input_offsets[inner_dim_idx] + output_inner_pad_before_size), Index(0)));
eigen_assert(output_inner_copy_size >= 0);
// How many values to fill with padding AFTER reading from the input inner
// dimension.
const Index output_inner_pad_after_size =
(output_inner_dim_size - output_inner_copy_size -
output_inner_pad_before_size);
(output_inner_dim_size - output_inner_copy_size - output_inner_pad_before_size);
// Sanity check, sum of all sizes must be equal to the output size.
eigen_assert(output_inner_dim_size ==
(output_inner_pad_before_size + output_inner_copy_size +
output_inner_pad_after_size));
(output_inner_pad_before_size + output_inner_copy_size + output_inner_pad_after_size));
// Keep track of current coordinates and padding in the output.
DSizes<Index, NumDims> output_coord = output_offsets;
@@ -358,8 +327,7 @@ struct TensorEvaluator<const TensorPaddingOp<PaddingDimensions, ArgType>, Device
typedef internal::StridedLinearBufferCopy<ScalarNoConst, Index> LinCopy;
// Prepare storage for the materialized padding result.
const typename TensorBlock::Storage block_storage =
TensorBlock::prepareStorage(desc, scratch);
const typename TensorBlock::Storage block_storage = TensorBlock::prepareStorage(desc, scratch);
// TODO(ezhulenev): Squeeze multiple non-padded inner dimensions into a
// single logical inner dimension.
@@ -367,12 +335,11 @@ struct TensorEvaluator<const TensorPaddingOp<PaddingDimensions, ArgType>, Device
// When possible we squeeze writes for the innermost (only if non-padded)
// dimension with the first padded dimension. This allows to reduce the
// number of calls to LinCopy and better utilize vector instructions.
const bool squeeze_writes =
NumDims > 1 &&
// inner dimension is not padded
(input_inner_dim_size == m_dimensions[inner_dim_idx]) &&
// and equal to the block inner dimension
(input_inner_dim_size == output_inner_dim_size);
const bool squeeze_writes = NumDims > 1 &&
// inner dimension is not padded
(input_inner_dim_size == m_dimensions[inner_dim_idx]) &&
// and equal to the block inner dimension
(input_inner_dim_size == output_inner_dim_size);
const int squeeze_dim = IsColMajor ? inner_dim_idx + 1 : inner_dim_idx - 1;
@@ -380,11 +347,9 @@ struct TensorEvaluator<const TensorPaddingOp<PaddingDimensions, ArgType>, Device
const Index squeeze_max_coord =
squeeze_writes ? numext::mini(
// max non-padded element in the input
static_cast<Index>(m_dimensions[squeeze_dim] -
m_padding[squeeze_dim].second),
static_cast<Index>(m_dimensions[squeeze_dim] - m_padding[squeeze_dim].second),
// max element in the output buffer
static_cast<Index>(output_offsets[squeeze_dim] +
desc.dimension(squeeze_dim)))
static_cast<Index>(output_offsets[squeeze_dim] + desc.dimension(squeeze_dim)))
: static_cast<Index>(0);
// Iterate copying data from `m_impl.data()` to the output buffer.
@@ -401,11 +366,9 @@ struct TensorEvaluator<const TensorPaddingOp<PaddingDimensions, ArgType>, Device
// Fill single innermost dimension with padding value.
size += output_inner_dim_size;
LinCopy::template Run<LinCopy::Kind::FillLinear>(
typename LinCopy::Dst(output_offset, 1, block_storage.data()),
typename LinCopy::Src(0, 0, &m_paddingValue),
output_inner_dim_size);
LinCopy::template Run<LinCopy::Kind::FillLinear>(typename LinCopy::Dst(output_offset, 1, block_storage.data()),
typename LinCopy::Src(0, 0, &m_paddingValue),
output_inner_dim_size);
} else if (squeeze_writes) {
// Squeeze multiple reads from innermost dimensions.
@@ -413,10 +376,9 @@ struct TensorEvaluator<const TensorPaddingOp<PaddingDimensions, ArgType>, Device
size += output_inner_dim_size * squeeze_num;
// Copy `squeeze_num` inner dimensions from input to output.
LinCopy::template Run<LinCopy::Kind::Linear>(
typename LinCopy::Dst(output_offset, 1, block_storage.data()),
typename LinCopy::Src(input_offset, 1, m_impl.data()),
output_inner_dim_size * squeeze_num);
LinCopy::template Run<LinCopy::Kind::Linear>(typename LinCopy::Dst(output_offset, 1, block_storage.data()),
typename LinCopy::Src(input_offset, 1, m_impl.data()),
output_inner_dim_size * squeeze_num);
// Update iteration state for only `squeeze_num - 1` processed inner
// dimensions, because we have another iteration state update at the end
@@ -434,10 +396,9 @@ struct TensorEvaluator<const TensorPaddingOp<PaddingDimensions, ArgType>, Device
{ // Fill with padding before copying from input inner dimension.
const Index out = output_offset;
LinCopy::template Run<LinCopy::Kind::FillLinear>(
typename LinCopy::Dst(out, 1, block_storage.data()),
typename LinCopy::Src(0, 0, &m_paddingValue),
output_inner_pad_before_size);
LinCopy::template Run<LinCopy::Kind::FillLinear>(typename LinCopy::Dst(out, 1, block_storage.data()),
typename LinCopy::Src(0, 0, &m_paddingValue),
output_inner_pad_before_size);
}
{ // Copy data from input inner dimension.
@@ -446,20 +407,17 @@ struct TensorEvaluator<const TensorPaddingOp<PaddingDimensions, ArgType>, Device
eigen_assert(output_inner_copy_size == 0 || m_impl.data() != NULL);
LinCopy::template Run<LinCopy::Kind::Linear>(
typename LinCopy::Dst(out, 1, block_storage.data()),
typename LinCopy::Src(in, 1, m_impl.data()),
output_inner_copy_size);
LinCopy::template Run<LinCopy::Kind::Linear>(typename LinCopy::Dst(out, 1, block_storage.data()),
typename LinCopy::Src(in, 1, m_impl.data()),
output_inner_copy_size);
}
{ // Fill with padding after copying from input inner dimension.
const Index out = output_offset + output_inner_pad_before_size +
output_inner_copy_size;
const Index out = output_offset + output_inner_pad_before_size + output_inner_copy_size;
LinCopy::template Run<LinCopy::Kind::FillLinear>(
typename LinCopy::Dst(out, 1, block_storage.data()),
typename LinCopy::Src(0, 0, &m_paddingValue),
output_inner_pad_after_size);
LinCopy::template Run<LinCopy::Kind::FillLinear>(typename LinCopy::Dst(out, 1, block_storage.data()),
typename LinCopy::Src(0, 0, &m_paddingValue),
output_inner_pad_after_size);
}
}
@@ -488,13 +446,7 @@ struct TensorEvaluator<const TensorPaddingOp<PaddingDimensions, ArgType>, Device
private:
struct BlockIteratorState {
BlockIteratorState()
: count(0),
size(0),
input_stride(0),
input_span(0),
output_stride(0),
output_span(0) {}
BlockIteratorState() : count(0), size(0), input_stride(0), input_span(0), output_stride(0), output_span(0) {}
Index count;
Index size;
@@ -504,48 +456,39 @@ struct TensorEvaluator<const TensorPaddingOp<PaddingDimensions, ArgType>, Device
Index output_span;
};
EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE bool isPaddingAtIndexForDim(
Index index, int dim_index) const {
EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE bool isPaddingAtIndexForDim(Index index, int dim_index) const {
return (!internal::index_pair_first_statically_eq<PaddingDimensions>(dim_index, 0) &&
index < m_padding[dim_index].first) ||
(!internal::index_pair_second_statically_eq<PaddingDimensions>(dim_index, 0) &&
index >= m_dimensions[dim_index] - m_padding[dim_index].second);
(!internal::index_pair_second_statically_eq<PaddingDimensions>(dim_index, 0) &&
index >= m_dimensions[dim_index] - m_padding[dim_index].second);
}
EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE bool isLeftPaddingCompileTimeZero(
int dim_index) const {
EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE bool isLeftPaddingCompileTimeZero(int dim_index) const {
return internal::index_pair_first_statically_eq<PaddingDimensions>(dim_index, 0);
}
EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE bool isRightPaddingCompileTimeZero(
int dim_index) const {
EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE bool isRightPaddingCompileTimeZero(int dim_index) const {
return internal::index_pair_second_statically_eq<PaddingDimensions>(dim_index, 0);
}
void updateCostPerDimension(TensorOpCost& cost, int i, bool first) const {
const double in = static_cast<double>(m_impl.dimensions()[i]);
const double out = in + m_padding[i].first + m_padding[i].second;
if (out == 0)
return;
if (out == 0) return;
const double reduction = in / out;
cost *= reduction;
if (first) {
cost += TensorOpCost(0, 0, 2 * TensorOpCost::AddCost<Index>() +
reduction * (1 * TensorOpCost::AddCost<Index>()));
cost += TensorOpCost(0, 0, 2 * TensorOpCost::AddCost<Index>() + reduction * (1 * TensorOpCost::AddCost<Index>()));
} else {
cost += TensorOpCost(0, 0, 2 * TensorOpCost::AddCost<Index>() +
2 * TensorOpCost::MulCost<Index>() +
reduction * (2 * TensorOpCost::MulCost<Index>() +
1 * TensorOpCost::DivCost<Index>()));
cost += TensorOpCost(0, 0,
2 * TensorOpCost::AddCost<Index>() + 2 * TensorOpCost::MulCost<Index>() +
reduction * (2 * TensorOpCost::MulCost<Index>() + 1 * TensorOpCost::DivCost<Index>()));
}
}
protected:
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketReturnType packetColMajor(Index index) const
{
eigen_assert(index+PacketSize-1 < dimensions().TotalSize());
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketReturnType packetColMajor(Index index) const {
eigen_assert(index + PacketSize - 1 < dimensions().TotalSize());
const Index initialIndex = index;
Index inputIndex = 0;
@@ -555,23 +498,21 @@ struct TensorEvaluator<const TensorPaddingOp<PaddingDimensions, ArgType>, Device
const Index lastIdx = index + PacketSize - 1;
const Index lastPaddedLeft = m_padding[i].first * m_outputStrides[i];
const Index firstPaddedRight = (m_dimensions[i] - m_padding[i].second) * m_outputStrides[i];
const Index lastPaddedRight = m_outputStrides[i+1];
const Index lastPaddedRight = m_outputStrides[i + 1];
if (!isLeftPaddingCompileTimeZero(i) && lastIdx < lastPaddedLeft) {
// all the coefficient are in the padding zone.
return internal::pset1<PacketReturnType>(m_paddingValue);
}
else if (!isRightPaddingCompileTimeZero(i) && firstIdx >= firstPaddedRight && lastIdx < lastPaddedRight) {
} else if (!isRightPaddingCompileTimeZero(i) && firstIdx >= firstPaddedRight && lastIdx < lastPaddedRight) {
// all the coefficient are in the padding zone.
return internal::pset1<PacketReturnType>(m_paddingValue);
}
else if ((isLeftPaddingCompileTimeZero(i) && isRightPaddingCompileTimeZero(i)) || (firstIdx >= lastPaddedLeft && lastIdx < firstPaddedRight)) {
} else if ((isLeftPaddingCompileTimeZero(i) && isRightPaddingCompileTimeZero(i)) ||
(firstIdx >= lastPaddedLeft && lastIdx < firstPaddedRight)) {
// all the coefficient are between the 2 padding zones.
const Index idx = index / m_outputStrides[i];
inputIndex += (idx - m_padding[i].first) * m_inputStrides[i];
index -= idx * m_outputStrides[i];
}
else {
} else {
// Every other case
return packetWithPossibleZero(initialIndex);
}
@@ -586,12 +527,11 @@ struct TensorEvaluator<const TensorPaddingOp<PaddingDimensions, ArgType>, Device
if (!isLeftPaddingCompileTimeZero(0) && lastIdx < lastPaddedLeft) {
// all the coefficient are in the padding zone.
return internal::pset1<PacketReturnType>(m_paddingValue);
}
else if (!isRightPaddingCompileTimeZero(0) && firstIdx >= firstPaddedRight && lastIdx < lastPaddedRight) {
} else if (!isRightPaddingCompileTimeZero(0) && firstIdx >= firstPaddedRight && lastIdx < lastPaddedRight) {
// all the coefficient are in the padding zone.
return internal::pset1<PacketReturnType>(m_paddingValue);
}
else if ((isLeftPaddingCompileTimeZero(0) && isRightPaddingCompileTimeZero(0)) || (firstIdx >= lastPaddedLeft && lastIdx < firstPaddedRight)) {
} else if ((isLeftPaddingCompileTimeZero(0) && isRightPaddingCompileTimeZero(0)) ||
(firstIdx >= lastPaddedLeft && lastIdx < firstPaddedRight)) {
// all the coefficient are between the 2 padding zones.
inputIndex += (index - m_padding[0].first);
return m_impl.template packet<Unaligned>(inputIndex);
@@ -600,9 +540,8 @@ struct TensorEvaluator<const TensorPaddingOp<PaddingDimensions, ArgType>, Device
return packetWithPossibleZero(initialIndex);
}
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketReturnType packetRowMajor(Index index) const
{
eigen_assert(index+PacketSize-1 < dimensions().TotalSize());
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketReturnType packetRowMajor(Index index) const {
eigen_assert(index + PacketSize - 1 < dimensions().TotalSize());
const Index initialIndex = index;
Index inputIndex = 0;
@@ -610,25 +549,23 @@ struct TensorEvaluator<const TensorPaddingOp<PaddingDimensions, ArgType>, Device
for (int i = 0; i < NumDims - 1; ++i) {
const Index firstIdx = index;
const Index lastIdx = index + PacketSize - 1;
const Index lastPaddedLeft = m_padding[i].first * m_outputStrides[i+1];
const Index firstPaddedRight = (m_dimensions[i] - m_padding[i].second) * m_outputStrides[i+1];
const Index lastPaddedLeft = m_padding[i].first * m_outputStrides[i + 1];
const Index firstPaddedRight = (m_dimensions[i] - m_padding[i].second) * m_outputStrides[i + 1];
const Index lastPaddedRight = m_outputStrides[i];
if (!isLeftPaddingCompileTimeZero(i) && lastIdx < lastPaddedLeft) {
// all the coefficient are in the padding zone.
return internal::pset1<PacketReturnType>(m_paddingValue);
}
else if (!isRightPaddingCompileTimeZero(i) && firstIdx >= firstPaddedRight && lastIdx < lastPaddedRight) {
} else if (!isRightPaddingCompileTimeZero(i) && firstIdx >= firstPaddedRight && lastIdx < lastPaddedRight) {
// all the coefficient are in the padding zone.
return internal::pset1<PacketReturnType>(m_paddingValue);
}
else if ((isLeftPaddingCompileTimeZero(i) && isRightPaddingCompileTimeZero(i)) || (firstIdx >= lastPaddedLeft && lastIdx < firstPaddedRight)) {
} else if ((isLeftPaddingCompileTimeZero(i) && isRightPaddingCompileTimeZero(i)) ||
(firstIdx >= lastPaddedLeft && lastIdx < firstPaddedRight)) {
// all the coefficient are between the 2 padding zones.
const Index idx = index / m_outputStrides[i+1];
const Index idx = index / m_outputStrides[i + 1];
inputIndex += (idx - m_padding[i].first) * m_inputStrides[i];
index -= idx * m_outputStrides[i+1];
}
else {
index -= idx * m_outputStrides[i + 1];
} else {
// Every other case
return packetWithPossibleZero(initialIndex);
}
@@ -636,40 +573,39 @@ struct TensorEvaluator<const TensorPaddingOp<PaddingDimensions, ArgType>, Device
const Index lastIdx = index + PacketSize - 1;
const Index firstIdx = index;
const Index lastPaddedLeft = m_padding[NumDims-1].first;
const Index firstPaddedRight = (m_dimensions[NumDims-1] - m_padding[NumDims-1].second);
const Index lastPaddedRight = m_outputStrides[NumDims-1];
const Index lastPaddedLeft = m_padding[NumDims - 1].first;
const Index firstPaddedRight = (m_dimensions[NumDims - 1] - m_padding[NumDims - 1].second);
const Index lastPaddedRight = m_outputStrides[NumDims - 1];
if (!isLeftPaddingCompileTimeZero(NumDims-1) && lastIdx < lastPaddedLeft) {
if (!isLeftPaddingCompileTimeZero(NumDims - 1) && lastIdx < lastPaddedLeft) {
// all the coefficient are in the padding zone.
return internal::pset1<PacketReturnType>(m_paddingValue);
}
else if (!isRightPaddingCompileTimeZero(NumDims-1) && firstIdx >= firstPaddedRight && lastIdx < lastPaddedRight) {
} else if (!isRightPaddingCompileTimeZero(NumDims - 1) && firstIdx >= firstPaddedRight &&
lastIdx < lastPaddedRight) {
// all the coefficient are in the padding zone.
return internal::pset1<PacketReturnType>(m_paddingValue);
}
else if ((isLeftPaddingCompileTimeZero(NumDims-1) && isRightPaddingCompileTimeZero(NumDims-1)) || (firstIdx >= lastPaddedLeft && lastIdx < firstPaddedRight)) {
} else if ((isLeftPaddingCompileTimeZero(NumDims - 1) && isRightPaddingCompileTimeZero(NumDims - 1)) ||
(firstIdx >= lastPaddedLeft && lastIdx < firstPaddedRight)) {
// all the coefficient are between the 2 padding zones.
inputIndex += (index - m_padding[NumDims-1].first);
inputIndex += (index - m_padding[NumDims - 1].first);
return m_impl.template packet<Unaligned>(inputIndex);
}
// Every other case
return packetWithPossibleZero(initialIndex);
}
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketReturnType packetWithPossibleZero(Index index) const
{
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketReturnType packetWithPossibleZero(Index index) const {
EIGEN_ALIGN_MAX std::remove_const_t<CoeffReturnType> values[PacketSize];
EIGEN_UNROLL_LOOP
for (int i = 0; i < PacketSize; ++i) {
values[i] = coeff(index+i);
values[i] = coeff(index + i);
}
PacketReturnType rslt = internal::pload<PacketReturnType>(values);
return rslt;
}
Dimensions m_dimensions;
array<Index, NumDims+1> m_outputStrides;
array<Index, NumDims + 1> m_outputStrides;
array<Index, NumDims> m_inputStrides;
TensorEvaluator<ArgType, Device> m_impl;
PaddingDimensions m_padding;
@@ -679,9 +615,6 @@ struct TensorEvaluator<const TensorPaddingOp<PaddingDimensions, ArgType>, Device
const Device EIGEN_DEVICE_REF m_device;
};
} // end namespace Eigen
} // end namespace Eigen
#endif // EIGEN_CXX11_TENSOR_TENSOR_PADDING_H
#endif // EIGEN_CXX11_TENSOR_TENSOR_PADDING_H