devtools/eigen
1
0
Fork 0
mirror of https://gitlab.com/libeigen/eigen.git synced 2026-04-10 11:34:33 +08:00
Code Issues Packages Projects Releases Wiki Activity

Fix GEBP half/quarter-packet loops for nr>=8 RHS packing on ARM64

Browse Source 
libeigen/eigen!2216

Co-authored-by: Rasmus Munk Larsen <rmlarsen@gmail.com>
This commit is contained in:
Rasmus Munk Larsen
2026-02-25 19:26:49 -08:00
parent 6b6d0d8c8e
commit d8ed4f6884
1 changed files with 20 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
Eigen/src/Core/products/GeneralBlockPanelKernel.h
View File

@@ -1496,7 +1496,16 @@ EIGEN_DONT_INLINE void gebp_kernel<LhsScalar, RhsScalar, Index, DataMapper, mr,
EIGEN_IF_CONSTEXPR((LhsProgressHalf < LhsProgress) && mr >= LhsProgressHalf) { EIGEN_IF_CONSTEXPR((LhsProgressHalf < LhsProgress) && mr >= LhsProgressHalf) {
HalfTraits half_traits; HalfTraits half_traits;
for (Index i = peeled_mc1; i < peeled_mc_half; i += LhsProgressHalf) { for (Index i = peeled_mc1; i < peeled_mc_half; i += LhsProgressHalf) {
for (Index j2 = 0; j2 < packet_cols4; j2 += 4) { #if EIGEN_ARCH_ARM64 || EIGEN_ARCH_LOONGARCH64
EIGEN_IF_CONSTEXPR(nr >= 8) {
for (Index j2 = 0; j2 < packet_cols8; j2 += 8) {
gebp_micro_panel_impl<1, 8, HalfTraits, LhsScalar, RhsScalar, ResScalar, Index, DataMapper, LinearMapper,
LhsPacket>(half_traits, res, blockA, blockB, alpha, i, j2, depth, strideA, strideB,
offsetA, offsetB, prefetch_res_offset, peeled_kc, pk);
}
}
#endif
for (Index j2 = packet_cols8; j2 < packet_cols4; j2 += 4) {
gebp_micro_panel_impl<1, 4, HalfTraits, LhsScalar, RhsScalar, ResScalar, Index, DataMapper, LinearMapper, gebp_micro_panel_impl<1, 4, HalfTraits, LhsScalar, RhsScalar, ResScalar, Index, DataMapper, LinearMapper,
LhsPacket>(half_traits, res, blockA, blockB, alpha, i, j2, depth, strideA, strideB, LhsPacket>(half_traits, res, blockA, blockB, alpha, i, j2, depth, strideA, strideB,
offsetA, offsetB, prefetch_res_offset, peeled_kc, pk); offsetA, offsetB, prefetch_res_offset, peeled_kc, pk);
@@ -1513,7 +1522,16 @@ EIGEN_DONT_INLINE void gebp_kernel<LhsScalar, RhsScalar, Index, DataMapper, mr,
EIGEN_IF_CONSTEXPR((LhsProgressQuarter < LhsProgressHalf) && mr >= LhsProgressQuarter) { EIGEN_IF_CONSTEXPR((LhsProgressQuarter < LhsProgressHalf) && mr >= LhsProgressQuarter) {
QuarterTraits quarter_traits; QuarterTraits quarter_traits;
for (Index i = peeled_mc_half; i < peeled_mc_quarter; i += LhsProgressQuarter) { for (Index i = peeled_mc_half; i < peeled_mc_quarter; i += LhsProgressQuarter) {
for (Index j2 = 0; j2 < packet_cols4; j2 += 4) { #if EIGEN_ARCH_ARM64 || EIGEN_ARCH_LOONGARCH64
EIGEN_IF_CONSTEXPR(nr >= 8) {
for (Index j2 = 0; j2 < packet_cols8; j2 += 8) {
gebp_micro_panel_impl<1, 8, QuarterTraits, LhsScalar, RhsScalar, ResScalar, Index, DataMapper, LinearMapper,
LhsPacket>(quarter_traits, res, blockA, blockB, alpha, i, j2, depth, strideA, strideB,
offsetA, offsetB, prefetch_res_offset, peeled_kc, pk);
}
}
#endif
for (Index j2 = packet_cols8; j2 < packet_cols4; j2 += 4) {
gebp_micro_panel_impl<1, 4, QuarterTraits, LhsScalar, RhsScalar, ResScalar, Index, DataMapper, LinearMapper, gebp_micro_panel_impl<1, 4, QuarterTraits, LhsScalar, RhsScalar, ResScalar, Index, DataMapper, LinearMapper,
LhsPacket>(quarter_traits, res, blockA, blockB, alpha, i, j2, depth, strideA, strideB, LhsPacket>(quarter_traits, res, blockA, blockB, alpha, i, j2, depth, strideA, strideB,
offsetA, offsetB, prefetch_res_offset, peeled_kc, pk); offsetA, offsetB, prefetch_res_offset, peeled_kc, pk);