devtools/eigen
1
0
Fork 0
mirror of https://gitlab.com/libeigen/eigen.git synced 2026-04-10 11:34:33 +08:00
Code Issues Packages Projects Releases Wiki Activity

Fix CUDA+Clang build warnings.

Browse Source 
libeigen/eigen!2241
This commit is contained in:
Antonio Sánchez
2026-03-04 09:41:01 +00:00
committed by Rasmus Munk Larsen
parent 0269c017aa
commit abc3d6014d
11 changed files with 356 additions and 334 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
CMakeLists.txt
View File

@@ -414,6 +414,7 @@ if (EIGEN_BUILD_TESTING)
ei_add_cxx_compiler_flag("-Wno-psabi") ei_add_cxx_compiler_flag("-Wno-psabi")
ei_add_cxx_compiler_flag("-Wno-variadic-macros") ei_add_cxx_compiler_flag("-Wno-variadic-macros")
ei_add_cxx_compiler_flag("-Wno-long-long") ei_add_cxx_compiler_flag("-Wno-long-long")
ei_add_cxx_compiler_flag("-Wno-pass-failed") # disable clang's warning for unrolling when the loop count is dynamic.
ei_add_cxx_compiler_flag("-fno-common") ei_add_cxx_compiler_flag("-fno-common")
ei_add_cxx_compiler_flag("-fstrict-aliasing") ei_add_cxx_compiler_flag("-fstrict-aliasing")
ei_add_cxx_compiler_flag("-wd981") # disable ICC's "operands are evaluated in unspecified order" remark ei_add_cxx_compiler_flag("-wd981") # disable ICC's "operands are evaluated in unspecified order" remark

18
Eigen/src/Core/products/GeneralBlockPanelKernel.h
View File

@@ -151,13 +151,13 @@ void evaluateProductBlockingSizesHeuristic(Index& k, Index& m, Index& n, Index n
// increasing the value of k, so we'll cap it at 320 (value determined // increasing the value of k, so we'll cap it at 320 (value determined
// experimentally). // experimentally).
// To avoid that k vanishes, we make k_cache at least as big as kr // To avoid that k vanishes, we make k_cache at least as big as kr
const Index k_cache = numext::maxi<Index>(kr, (numext::mini<Index>)((l1 - ksub) / kdiv, 320)); const Index k_cache = numext::maxi<Index>(kr, (numext::mini<Index>)(static_cast<Index>((l1 - ksub) / kdiv), 320));
if (k_cache < k) { if (k_cache < k) {
k = k_cache - (k_cache % kr); k = k_cache - (k_cache % kr);
eigen_internal_assert(k > 0); eigen_internal_assert(k > 0);
} }
const Index n_cache = (l2 - l1) / (nr * sizeof(RhsScalar) * k); const Index n_cache = static_cast<Index>((l2 - l1) / (nr * sizeof(RhsScalar) * k));
const Index n_per_thread = numext::div_ceil(n, num_threads); const Index n_per_thread = numext::div_ceil(n, num_threads);
if (n_cache <= n_per_thread) { if (n_cache <= n_per_thread) {
// Don't exceed the capacity of the l2 cache. // Don't exceed the capacity of the l2 cache.
@@ -170,7 +170,7 @@ void evaluateProductBlockingSizesHeuristic(Index& k, Index& m, Index& n, Index n
if (l3 > l2) { if (l3 > l2) {
// l3 is shared between all cores, so we'll give each thread its own chunk of l3. // l3 is shared between all cores, so we'll give each thread its own chunk of l3.
const Index m_cache = (l3 - l2) / (sizeof(LhsScalar) * k * num_threads); const Index m_cache = static_cast<Index>((l3 - l2) / (sizeof(LhsScalar) * k * num_threads));
const Index m_per_thread = numext::div_ceil(m, num_threads); const Index m_per_thread = numext::div_ceil(m, num_threads);
if (m_cache < m_per_thread && m_cache >= static_cast<Index>(mr)) { if (m_cache < m_per_thread && m_cache >= static_cast<Index>(mr)) {
m = m_cache - (m_cache % mr); m = m_cache - (m_cache % mr);
@@ -208,7 +208,7 @@ void evaluateProductBlockingSizesHeuristic(Index& k, Index& m, Index& n, Index n
// We also include a register-level block of the result (mx x nr). // We also include a register-level block of the result (mx x nr).
// (In an ideal world only the lhs panel would stay in L1) // (In an ideal world only the lhs panel would stay in L1)
// Moreover, kc has to be a multiple of 8 to be compatible with loop peeling, leading to a maximum blocking size of: // Moreover, kc has to be a multiple of 8 to be compatible with loop peeling, leading to a maximum blocking size of:
const Index max_kc = numext::maxi<Index>(((l1 - k_sub) / k_div) & (~(k_peeling - 1)), 1); const Index max_kc = numext::maxi<Index>(static_cast<Index>(((l1 - k_sub) / k_div) & (~(k_peeling - 1))), 1);
const Index old_k = k; const Index old_k = k;
if (k > max_kc) { if (k > max_kc) {
// We are really blocking on the third dimension: // We are really blocking on the third dimension:
@@ -242,9 +242,9 @@ void evaluateProductBlockingSizesHeuristic(Index& k, Index& m, Index& n, Index n
// that spills to L3 but remains accessible with low latency. This matches // that spills to L3 but remains accessible with low latency. This matches
// the empirically-tuned constant (1.5MB) previously used when L2 was 1MB. // the empirically-tuned constant (1.5MB) previously used when L2 was 1MB.
#ifdef EIGEN_DEBUG_SMALL_PRODUCT_BLOCKS #ifdef EIGEN_DEBUG_SMALL_PRODUCT_BLOCKS
const Index actual_l2 = l3; const Index actual_l2 = static_cast<Index>(l3);
#else #else
const Index actual_l2 = l2 * 3 / 2; const Index actual_l2 = static_cast<Index>(l2 * 3 / 2);
#endif #endif
// Here, nc is chosen such that a block of kc x nc of the rhs fit within half of L2. // Here, nc is chosen such that a block of kc x nc of the rhs fit within half of L2.
@@ -255,7 +255,7 @@ void evaluateProductBlockingSizesHeuristic(Index& k, Index& m, Index& n, Index n
// and it becomes fruitful to keep the packed rhs blocks in L1 if there is enough remaining space. // and it becomes fruitful to keep the packed rhs blocks in L1 if there is enough remaining space.
Index max_nc; Index max_nc;
const Index lhs_bytes = m * k * sizeof(LhsScalar); const Index lhs_bytes = m * k * sizeof(LhsScalar);
const Index remaining_l1 = l1 - k_sub - lhs_bytes; const Index remaining_l1 = static_cast<Index>(l1 - k_sub - lhs_bytes);
if (remaining_l1 >= Index(Traits::nr * sizeof(RhsScalar)) * k) { if (remaining_l1 >= Index(Traits::nr * sizeof(RhsScalar)) * k) {
// L1 blocking // L1 blocking
max_nc = remaining_l1 / (k * sizeof(RhsScalar)); max_nc = remaining_l1 / (k * sizeof(RhsScalar));
@@ -282,11 +282,11 @@ void evaluateProductBlockingSizesHeuristic(Index& k, Index& m, Index& n, Index n
if (problem_size <= 1024) { if (problem_size <= 1024) {
// problem is small enough to keep in L1 // problem is small enough to keep in L1
// Let's choose m such that lhs's block fit in 1/3 of L1 // Let's choose m such that lhs's block fit in 1/3 of L1
actual_lm = l1; actual_lm = static_cast<Index>(l1);
} else if (l3 != 0 && problem_size <= 32768) { } else if (l3 != 0 && problem_size <= 32768) {
// we have both L2 and L3, and problem is small enough to be kept in L2 // we have both L2 and L3, and problem is small enough to be kept in L2
// Let's choose m such that lhs's block fit in 1/3 of L2 // Let's choose m such that lhs's block fit in 1/3 of L2
actual_lm = l2; actual_lm = static_cast<Index>(l2);
max_mc = (numext::mini<Index>)(576, max_mc); max_mc = (numext::mini<Index>)(576, max_mc);
} }
Index mc = (numext::mini<Index>)(actual_lm / (3 * k * sizeof(LhsScalar)), max_mc); Index mc = (numext::mini<Index>)(actual_lm / (3 * k * sizeof(LhsScalar)), max_mc);

2
test/gpu_example.cu
View File

@@ -68,7 +68,7 @@ struct MultiplyKernel {
}; };
template <typename T1, typename T2, typename T3> template <typename T1, typename T2, typename T3>
void test_multiply(const T1& type1, const T2& type2, const T3& type3) { void test_multiply(const T1& type1, const T2& type2, const T3& /*type3*/) {
const T1 A = T1::Random(type1.rows(), type1.cols()); const T1 A = T1::Random(type1.rows(), type1.cols());
const T2 B = T2::Random(type2.rows(), type2.cols()); const T2 B = T2::Random(type2.rows(), type2.cols());
T3 C; T3 C;

613
unsupported/Eigen/src/Tensor/TensorContractionGpu.h
View File

@@ -30,6 +30,9 @@ __device__ EIGEN_STRONG_INLINE void EigenContractionKernelInternal(const LhsMapp
const Index base_m = 64 * m_block_idx; const Index base_m = 64 * m_block_idx;
const Index base_n = 64 * n_block_idx; const Index base_n = 64 * n_block_idx;
const Index thread_x = threadIdx.x;
const Index thread_y = threadIdx.y;
const Index thread_z = threadIdx.z;
// declare and initialize 64 registers for output 8x8 block // declare and initialize 64 registers for output 8x8 block
@@ -66,8 +69,8 @@ __device__ EIGEN_STRONG_INLINE void EigenContractionKernelInternal(const LhsMapp
// conflicts on writes and also none on reads. // conflicts on writes and also none on reads.
// storage indices // storage indices
const Index lhs_store_idx_base = threadIdx.y * 72 + threadIdx.x * 9 + threadIdx.z; const Index lhs_store_idx_base = thread_y * 72 + thread_x * 9 + thread_z;
const Index rhs_store_idx_base = threadIdx.y * 72 + threadIdx.z * 8 + threadIdx.x; const Index rhs_store_idx_base = thread_y * 72 + thread_z * 8 + thread_x;
const Index lhs_store_idx_0 = lhs_store_idx_base + 576 * 0; const Index lhs_store_idx_0 = lhs_store_idx_base + 576 * 0;
const Index lhs_store_idx_1 = lhs_store_idx_base + 576 * 1; const Index lhs_store_idx_1 = lhs_store_idx_base + 576 * 1;
@@ -88,151 +91,151 @@ __device__ EIGEN_STRONG_INLINE void EigenContractionKernelInternal(const LhsMapp
const Index rhs_store_idx_7 = rhs_store_idx_base + 576 * 7; const Index rhs_store_idx_7 = rhs_store_idx_base + 576 * 7;
// in the loading code, the following variables are important: // in the loading code, the following variables are important:
// threadIdx.x: the vertical position in an 8x8 block // thread_x: the vertical position in an 8x8 block
// threadIdx.y: the vertical index of the 8x8 block in the grid // thread_y: the vertical index of the 8x8 block in the grid
// threadIdx.z: the horizontal position in an 8x8 block // thread_z: the horizontal position in an 8x8 block
// k: the horizontal index of the 8x8 block in the grid // k: the horizontal index of the 8x8 block in the grid
// //
// The k parameter is implicit (it was the loop counter for a loop that went // The k parameter is implicit (it was the loop counter for a loop that went
// from 0 to <8, but now that loop is unrolled in the below code. // from 0 to <8, but now that loop is unrolled in the below code.
const Index load_idx_vert = threadIdx.x + 8 * threadIdx.y; const Index load_idx_vert = thread_x + 8 * thread_y;
const Index lhs_vert = base_m + load_idx_vert; const Index lhs_vert = base_m + load_idx_vert;
#define prefetchIntoRegisters(base_k) \ #define prefetchIntoRegisters(base_k) \
{ \ { \
lhs_pf0 = conv(0); \ lhs_pf0 = conv(0); \
lhs_pf1 = conv(0); \ lhs_pf1 = conv(0); \
lhs_pf2 = conv(0); \ lhs_pf2 = conv(0); \
lhs_pf3 = conv(0); \ lhs_pf3 = conv(0); \
lhs_pf4 = conv(0); \ lhs_pf4 = conv(0); \
lhs_pf5 = conv(0); \ lhs_pf5 = conv(0); \
lhs_pf6 = conv(0); \ lhs_pf6 = conv(0); \
lhs_pf7 = conv(0); \ lhs_pf7 = conv(0); \
\ \
rhs_pf0 = conv(0); \ rhs_pf0 = conv(0); \
rhs_pf1 = conv(0); \ rhs_pf1 = conv(0); \
rhs_pf2 = conv(0); \ rhs_pf2 = conv(0); \
rhs_pf3 = conv(0); \ rhs_pf3 = conv(0); \
rhs_pf4 = conv(0); \ rhs_pf4 = conv(0); \
rhs_pf5 = conv(0); \ rhs_pf5 = conv(0); \
rhs_pf6 = conv(0); \ rhs_pf6 = conv(0); \
rhs_pf7 = conv(0); \ rhs_pf7 = conv(0); \
\ \
if (!needs_edge_check || lhs_vert < m_size) { \ if (!needs_edge_check || lhs_vert < m_size) { \
const Index lhs_horiz_0 = base_k + threadIdx.z + 0 * 8; \ const Index lhs_horiz_0 = base_k + thread_z + 0 * 8; \
const Index lhs_horiz_1 = base_k + threadIdx.z + 1 * 8; \ const Index lhs_horiz_1 = base_k + thread_z + 1 * 8; \
const Index lhs_horiz_2 = base_k + threadIdx.z + 2 * 8; \ const Index lhs_horiz_2 = base_k + thread_z + 2 * 8; \
const Index lhs_horiz_3 = base_k + threadIdx.z + 3 * 8; \ const Index lhs_horiz_3 = base_k + thread_z + 3 * 8; \
const Index lhs_horiz_4 = base_k + threadIdx.z + 4 * 8; \ const Index lhs_horiz_4 = base_k + thread_z + 4 * 8; \
const Index lhs_horiz_5 = base_k + threadIdx.z + 5 * 8; \ const Index lhs_horiz_5 = base_k + thread_z + 5 * 8; \
const Index lhs_horiz_6 = base_k + threadIdx.z + 6 * 8; \ const Index lhs_horiz_6 = base_k + thread_z + 6 * 8; \
const Index lhs_horiz_7 = base_k + threadIdx.z + 7 * 8; \ const Index lhs_horiz_7 = base_k + thread_z + 7 * 8; \
\ \
if (!needs_edge_check || lhs_horiz_7 < k_size) { \ if (!needs_edge_check || lhs_horiz_7 < k_size) { \
lhs_pf0 = lhs(lhs_vert, lhs_horiz_0); \ lhs_pf0 = lhs(lhs_vert, lhs_horiz_0); \
lhs_pf1 = lhs(lhs_vert, lhs_horiz_1); \ lhs_pf1 = lhs(lhs_vert, lhs_horiz_1); \
lhs_pf2 = lhs(lhs_vert, lhs_horiz_2); \ lhs_pf2 = lhs(lhs_vert, lhs_horiz_2); \
lhs_pf3 = lhs(lhs_vert, lhs_horiz_3); \ lhs_pf3 = lhs(lhs_vert, lhs_horiz_3); \
lhs_pf4 = lhs(lhs_vert, lhs_horiz_4); \ lhs_pf4 = lhs(lhs_vert, lhs_horiz_4); \
lhs_pf5 = lhs(lhs_vert, lhs_horiz_5); \ lhs_pf5 = lhs(lhs_vert, lhs_horiz_5); \
lhs_pf6 = lhs(lhs_vert, lhs_horiz_6); \ lhs_pf6 = lhs(lhs_vert, lhs_horiz_6); \
lhs_pf7 = lhs(lhs_vert, lhs_horiz_7); \ lhs_pf7 = lhs(lhs_vert, lhs_horiz_7); \
} else if (lhs_horiz_6 < k_size) { \ } else if (lhs_horiz_6 < k_size) { \
lhs_pf0 = lhs(lhs_vert, lhs_horiz_0); \ lhs_pf0 = lhs(lhs_vert, lhs_horiz_0); \
lhs_pf1 = lhs(lhs_vert, lhs_horiz_1); \ lhs_pf1 = lhs(lhs_vert, lhs_horiz_1); \
lhs_pf2 = lhs(lhs_vert, lhs_horiz_2); \ lhs_pf2 = lhs(lhs_vert, lhs_horiz_2); \
lhs_pf3 = lhs(lhs_vert, lhs_horiz_3); \ lhs_pf3 = lhs(lhs_vert, lhs_horiz_3); \
lhs_pf4 = lhs(lhs_vert, lhs_horiz_4); \ lhs_pf4 = lhs(lhs_vert, lhs_horiz_4); \
lhs_pf5 = lhs(lhs_vert, lhs_horiz_5); \ lhs_pf5 = lhs(lhs_vert, lhs_horiz_5); \
lhs_pf6 = lhs(lhs_vert, lhs_horiz_6); \ lhs_pf6 = lhs(lhs_vert, lhs_horiz_6); \
} else if (lhs_horiz_5 < k_size) { \ } else if (lhs_horiz_5 < k_size) { \
lhs_pf0 = lhs(lhs_vert, lhs_horiz_0); \ lhs_pf0 = lhs(lhs_vert, lhs_horiz_0); \
lhs_pf1 = lhs(lhs_vert, lhs_horiz_1); \ lhs_pf1 = lhs(lhs_vert, lhs_horiz_1); \
lhs_pf2 = lhs(lhs_vert, lhs_horiz_2); \ lhs_pf2 = lhs(lhs_vert, lhs_horiz_2); \
lhs_pf3 = lhs(lhs_vert, lhs_horiz_3); \ lhs_pf3 = lhs(lhs_vert, lhs_horiz_3); \
lhs_pf4 = lhs(lhs_vert, lhs_horiz_4); \ lhs_pf4 = lhs(lhs_vert, lhs_horiz_4); \
lhs_pf5 = lhs(lhs_vert, lhs_horiz_5); \ lhs_pf5 = lhs(lhs_vert, lhs_horiz_5); \
} else if (lhs_horiz_4 < k_size) { \ } else if (lhs_horiz_4 < k_size) { \
lhs_pf0 = lhs(lhs_vert, lhs_horiz_0); \ lhs_pf0 = lhs(lhs_vert, lhs_horiz_0); \
lhs_pf1 = lhs(lhs_vert, lhs_horiz_1); \ lhs_pf1 = lhs(lhs_vert, lhs_horiz_1); \
lhs_pf2 = lhs(lhs_vert, lhs_horiz_2); \ lhs_pf2 = lhs(lhs_vert, lhs_horiz_2); \
lhs_pf3 = lhs(lhs_vert, lhs_horiz_3); \ lhs_pf3 = lhs(lhs_vert, lhs_horiz_3); \
lhs_pf4 = lhs(lhs_vert, lhs_horiz_4); \ lhs_pf4 = lhs(lhs_vert, lhs_horiz_4); \
} else if (lhs_horiz_3 < k_size) { \ } else if (lhs_horiz_3 < k_size) { \
lhs_pf0 = lhs(lhs_vert, lhs_horiz_0); \ lhs_pf0 = lhs(lhs_vert, lhs_horiz_0); \
lhs_pf1 = lhs(lhs_vert, lhs_horiz_1); \ lhs_pf1 = lhs(lhs_vert, lhs_horiz_1); \
lhs_pf2 = lhs(lhs_vert, lhs_horiz_2); \ lhs_pf2 = lhs(lhs_vert, lhs_horiz_2); \
lhs_pf3 = lhs(lhs_vert, lhs_horiz_3); \ lhs_pf3 = lhs(lhs_vert, lhs_horiz_3); \
} else if (lhs_horiz_2 < k_size) { \ } else if (lhs_horiz_2 < k_size) { \
lhs_pf0 = lhs(lhs_vert, lhs_horiz_0); \ lhs_pf0 = lhs(lhs_vert, lhs_horiz_0); \
lhs_pf1 = lhs(lhs_vert, lhs_horiz_1); \ lhs_pf1 = lhs(lhs_vert, lhs_horiz_1); \
lhs_pf2 = lhs(lhs_vert, lhs_horiz_2); \ lhs_pf2 = lhs(lhs_vert, lhs_horiz_2); \
} else if (lhs_horiz_1 < k_size) { \ } else if (lhs_horiz_1 < k_size) { \
lhs_pf0 = lhs(lhs_vert, lhs_horiz_0); \ lhs_pf0 = lhs(lhs_vert, lhs_horiz_0); \
lhs_pf1 = lhs(lhs_vert, lhs_horiz_1); \ lhs_pf1 = lhs(lhs_vert, lhs_horiz_1); \
} else if (lhs_horiz_0 < k_size) { \ } else if (lhs_horiz_0 < k_size) { \
lhs_pf0 = lhs(lhs_vert, lhs_horiz_0); \ lhs_pf0 = lhs(lhs_vert, lhs_horiz_0); \
} \ } \
} \ } \
\ \
const Index rhs_vert = base_k + load_idx_vert; \ const Index rhs_vert = base_k + load_idx_vert; \
if (!needs_edge_check || rhs_vert < k_size) { \ if (!needs_edge_check || rhs_vert < k_size) { \
const Index rhs_horiz_0 = base_n + threadIdx.z + 0 * 8; \ const Index rhs_horiz_0 = base_n + thread_z + 0 * 8; \
const Index rhs_horiz_1 = base_n + threadIdx.z + 1 * 8; \ const Index rhs_horiz_1 = base_n + thread_z + 1 * 8; \
const Index rhs_horiz_2 = base_n + threadIdx.z + 2 * 8; \ const Index rhs_horiz_2 = base_n + thread_z + 2 * 8; \
const Index rhs_horiz_3 = base_n + threadIdx.z + 3 * 8; \ const Index rhs_horiz_3 = base_n + thread_z + 3 * 8; \
const Index rhs_horiz_4 = base_n + threadIdx.z + 4 * 8; \ const Index rhs_horiz_4 = base_n + thread_z + 4 * 8; \
const Index rhs_horiz_5 = base_n + threadIdx.z + 5 * 8; \ const Index rhs_horiz_5 = base_n + thread_z + 5 * 8; \
const Index rhs_horiz_6 = base_n + threadIdx.z + 6 * 8; \ const Index rhs_horiz_6 = base_n + thread_z + 6 * 8; \
const Index rhs_horiz_7 = base_n + threadIdx.z + 7 * 8; \ const Index rhs_horiz_7 = base_n + thread_z + 7 * 8; \
\ \
if (rhs_horiz_7 < n_size) { \ if (rhs_horiz_7 < n_size) { \
rhs_pf0 = rhs(rhs_vert, rhs_horiz_0); \ rhs_pf0 = rhs(rhs_vert, rhs_horiz_0); \
rhs_pf1 = rhs(rhs_vert, rhs_horiz_1); \ rhs_pf1 = rhs(rhs_vert, rhs_horiz_1); \
rhs_pf2 = rhs(rhs_vert, rhs_horiz_2); \ rhs_pf2 = rhs(rhs_vert, rhs_horiz_2); \
rhs_pf3 = rhs(rhs_vert, rhs_horiz_3); \ rhs_pf3 = rhs(rhs_vert, rhs_horiz_3); \
rhs_pf4 = rhs(rhs_vert, rhs_horiz_4); \ rhs_pf4 = rhs(rhs_vert, rhs_horiz_4); \
rhs_pf5 = rhs(rhs_vert, rhs_horiz_5); \ rhs_pf5 = rhs(rhs_vert, rhs_horiz_5); \
rhs_pf6 = rhs(rhs_vert, rhs_horiz_6); \ rhs_pf6 = rhs(rhs_vert, rhs_horiz_6); \
rhs_pf7 = rhs(rhs_vert, rhs_horiz_7); \ rhs_pf7 = rhs(rhs_vert, rhs_horiz_7); \
} else if (rhs_horiz_6 < n_size) { \ } else if (rhs_horiz_6 < n_size) { \
rhs_pf0 = rhs(rhs_vert, rhs_horiz_0); \ rhs_pf0 = rhs(rhs_vert, rhs_horiz_0); \
rhs_pf1 = rhs(rhs_vert, rhs_horiz_1); \ rhs_pf1 = rhs(rhs_vert, rhs_horiz_1); \
rhs_pf2 = rhs(rhs_vert, rhs_horiz_2); \ rhs_pf2 = rhs(rhs_vert, rhs_horiz_2); \
rhs_pf3 = rhs(rhs_vert, rhs_horiz_3); \ rhs_pf3 = rhs(rhs_vert, rhs_horiz_3); \
rhs_pf4 = rhs(rhs_vert, rhs_horiz_4); \ rhs_pf4 = rhs(rhs_vert, rhs_horiz_4); \
rhs_pf5 = rhs(rhs_vert, rhs_horiz_5); \ rhs_pf5 = rhs(rhs_vert, rhs_horiz_5); \
rhs_pf6 = rhs(rhs_vert, rhs_horiz_6); \ rhs_pf6 = rhs(rhs_vert, rhs_horiz_6); \
} else if (rhs_horiz_5 < n_size) { \ } else if (rhs_horiz_5 < n_size) { \
rhs_pf0 = rhs(rhs_vert, rhs_horiz_0); \ rhs_pf0 = rhs(rhs_vert, rhs_horiz_0); \
rhs_pf1 = rhs(rhs_vert, rhs_horiz_1); \ rhs_pf1 = rhs(rhs_vert, rhs_horiz_1); \
rhs_pf2 = rhs(rhs_vert, rhs_horiz_2); \ rhs_pf2 = rhs(rhs_vert, rhs_horiz_2); \
rhs_pf3 = rhs(rhs_vert, rhs_horiz_3); \ rhs_pf3 = rhs(rhs_vert, rhs_horiz_3); \
rhs_pf4 = rhs(rhs_vert, rhs_horiz_4); \ rhs_pf4 = rhs(rhs_vert, rhs_horiz_4); \
rhs_pf5 = rhs(rhs_vert, rhs_horiz_5); \ rhs_pf5 = rhs(rhs_vert, rhs_horiz_5); \
} else if (rhs_horiz_4 < n_size) { \ } else if (rhs_horiz_4 < n_size) { \
rhs_pf0 = rhs(rhs_vert, rhs_horiz_0); \ rhs_pf0 = rhs(rhs_vert, rhs_horiz_0); \
rhs_pf1 = rhs(rhs_vert, rhs_horiz_1); \ rhs_pf1 = rhs(rhs_vert, rhs_horiz_1); \
rhs_pf2 = rhs(rhs_vert, rhs_horiz_2); \ rhs_pf2 = rhs(rhs_vert, rhs_horiz_2); \
rhs_pf3 = rhs(rhs_vert, rhs_horiz_3); \ rhs_pf3 = rhs(rhs_vert, rhs_horiz_3); \
rhs_pf4 = rhs(rhs_vert, rhs_horiz_4); \ rhs_pf4 = rhs(rhs_vert, rhs_horiz_4); \
} else if (rhs_horiz_3 < n_size) { \ } else if (rhs_horiz_3 < n_size) { \
rhs_pf0 = rhs(rhs_vert, rhs_horiz_0); \ rhs_pf0 = rhs(rhs_vert, rhs_horiz_0); \
rhs_pf1 = rhs(rhs_vert, rhs_horiz_1); \ rhs_pf1 = rhs(rhs_vert, rhs_horiz_1); \
rhs_pf2 = rhs(rhs_vert, rhs_horiz_2); \ rhs_pf2 = rhs(rhs_vert, rhs_horiz_2); \
rhs_pf3 = rhs(rhs_vert, rhs_horiz_3); \ rhs_pf3 = rhs(rhs_vert, rhs_horiz_3); \
} else if (rhs_horiz_2 < n_size) { \ } else if (rhs_horiz_2 < n_size) { \
rhs_pf0 = rhs(rhs_vert, rhs_horiz_0); \ rhs_pf0 = rhs(rhs_vert, rhs_horiz_0); \
rhs_pf1 = rhs(rhs_vert, rhs_horiz_1); \ rhs_pf1 = rhs(rhs_vert, rhs_horiz_1); \
rhs_pf2 = rhs(rhs_vert, rhs_horiz_2); \ rhs_pf2 = rhs(rhs_vert, rhs_horiz_2); \
} else if (rhs_horiz_1 < n_size) { \ } else if (rhs_horiz_1 < n_size) { \
rhs_pf0 = rhs(rhs_vert, rhs_horiz_0); \ rhs_pf0 = rhs(rhs_vert, rhs_horiz_0); \
rhs_pf1 = rhs(rhs_vert, rhs_horiz_1); \ rhs_pf1 = rhs(rhs_vert, rhs_horiz_1); \
} else if (rhs_horiz_0 < n_size) { \ } else if (rhs_horiz_0 < n_size) { \
rhs_pf0 = rhs(rhs_vert, rhs_horiz_0); \ rhs_pf0 = rhs(rhs_vert, rhs_horiz_0); \
} \ } \
} \ } \
} }
#define writeRegToShmem() \ #define writeRegToShmem() \
@@ -321,8 +324,8 @@ __device__ EIGEN_STRONG_INLINE void EigenContractionKernelInternal(const LhsMapp
Scalar rrow(7); Scalar rrow(7);
// Now x corresponds to k, y to m, and z to n // Now x corresponds to k, y to m, and z to n
const Scalar* lhs_block = &lhs_shmem[threadIdx.x + 9 * threadIdx.y]; const Scalar* lhs_block = &lhs_shmem[thread_x + 9 * thread_y];
const Scalar* rhs_block = &rhs_shmem[threadIdx.x + 8 * threadIdx.z]; const Scalar* rhs_block = &rhs_shmem[thread_x + 8 * thread_z];
#define lhs_element(i, j) lhs_block[72 * ((i) + 8 * (j))] #define lhs_element(i, j) lhs_block[72 * ((i) + 8 * (j))]
#define rhs_element(i, j) rhs_block[72 * ((i) + 8 * (j))] #define rhs_element(i, j) rhs_block[72 * ((i) + 8 * (j))]
@@ -441,7 +444,7 @@ __device__ EIGEN_STRONG_INLINE void EigenContractionKernelInternal(const LhsMapp
// wait for shared mem to be out of use // wait for shared mem to be out of use
__syncthreads(); __syncthreads();
#define writeResultShmem(i, j) lhs_shmem[i + 8 * threadIdx.y + 64 * threadIdx.z + 512 * j] = res(i, j); #define writeResultShmem(i, j) lhs_shmem[i + 8 * thread_y + 64 * thread_z + 512 * j] = res(i, j);
#define writeRow(i) \ #define writeRow(i) \
writeResultShmem(i, 0); \ writeResultShmem(i, 0); \
@@ -453,7 +456,7 @@ __device__ EIGEN_STRONG_INLINE void EigenContractionKernelInternal(const LhsMapp
writeResultShmem(i, 6); \ writeResultShmem(i, 6); \
writeResultShmem(i, 7); writeResultShmem(i, 7);
if (threadIdx.x == 0) { if (thread_x == 0) {
writeRow(0); writeRow(0);
writeRow(1); writeRow(1);
writeRow(2); writeRow(2);
@@ -466,34 +469,34 @@ __device__ EIGEN_STRONG_INLINE void EigenContractionKernelInternal(const LhsMapp
#undef writeResultShmem #undef writeResultShmem
#undef writeRow #undef writeRow
const int max_i_write = numext::mini((int)((m_size - base_m - threadIdx.y + 7) / 8), 8); const int max_i_write = numext::mini((int)((m_size - base_m - thread_y + 7) / 8), 8);
const int max_j_write = numext::mini((int)((n_size - base_n - threadIdx.z + 7) / 8), 8); const int max_j_write = numext::mini((int)((n_size - base_n - thread_z + 7) / 8), 8);
if (threadIdx.x < max_i_write) { if (thread_x < max_i_write) {
if (max_j_write == 8) { if (max_j_write == 8) {
// TODO: Can we trade bank conflicts for coalesced writes? // TODO: Can we trade bank conflicts for coalesced writes?
Scalar val0 = lhs_shmem[threadIdx.x + 8 * threadIdx.y + 64 * threadIdx.z + 512 * 0]; Scalar val0 = lhs_shmem[thread_x + 8 * thread_y + 64 * thread_z + 512 * 0];
Scalar val1 = lhs_shmem[threadIdx.x + 8 * threadIdx.y + 64 * threadIdx.z + 512 * 1]; Scalar val1 = lhs_shmem[thread_x + 8 * thread_y + 64 * thread_z + 512 * 1];
Scalar val2 = lhs_shmem[threadIdx.x + 8 * threadIdx.y + 64 * threadIdx.z + 512 * 2]; Scalar val2 = lhs_shmem[thread_x + 8 * thread_y + 64 * thread_z + 512 * 2];
Scalar val3 = lhs_shmem[threadIdx.x + 8 * threadIdx.y + 64 * threadIdx.z + 512 * 3]; Scalar val3 = lhs_shmem[thread_x + 8 * thread_y + 64 * thread_z + 512 * 3];
Scalar val4 = lhs_shmem[threadIdx.x + 8 * threadIdx.y + 64 * threadIdx.z + 512 * 4]; Scalar val4 = lhs_shmem[thread_x + 8 * thread_y + 64 * thread_z + 512 * 4];
Scalar val5 = lhs_shmem[threadIdx.x + 8 * threadIdx.y + 64 * threadIdx.z + 512 * 5]; Scalar val5 = lhs_shmem[thread_x + 8 * thread_y + 64 * thread_z + 512 * 5];
Scalar val6 = lhs_shmem[threadIdx.x + 8 * threadIdx.y + 64 * threadIdx.z + 512 * 6]; Scalar val6 = lhs_shmem[thread_x + 8 * thread_y + 64 * thread_z + 512 * 6];
Scalar val7 = lhs_shmem[threadIdx.x + 8 * threadIdx.y + 64 * threadIdx.z + 512 * 7]; Scalar val7 = lhs_shmem[thread_x + 8 * thread_y + 64 * thread_z + 512 * 7];
output(base_m + threadIdx.y + 8 * threadIdx.x, base_n + threadIdx.z + 8 * 0) = val0; output(base_m + thread_y + 8 * thread_x, base_n + thread_z + 8 * 0) = val0;
output(base_m + threadIdx.y + 8 * threadIdx.x, base_n + threadIdx.z + 8 * 1) = val1; output(base_m + thread_y + 8 * thread_x, base_n + thread_z + 8 * 1) = val1;
output(base_m + threadIdx.y + 8 * threadIdx.x, base_n + threadIdx.z + 8 * 2) = val2; output(base_m + thread_y + 8 * thread_x, base_n + thread_z + 8 * 2) = val2;
output(base_m + threadIdx.y + 8 * threadIdx.x, base_n + threadIdx.z + 8 * 3) = val3; output(base_m + thread_y + 8 * thread_x, base_n + thread_z + 8 * 3) = val3;
output(base_m + threadIdx.y + 8 * threadIdx.x, base_n + threadIdx.z + 8 * 4) = val4; output(base_m + thread_y + 8 * thread_x, base_n + thread_z + 8 * 4) = val4;
output(base_m + threadIdx.y + 8 * threadIdx.x, base_n + threadIdx.z + 8 * 5) = val5; output(base_m + thread_y + 8 * thread_x, base_n + thread_z + 8 * 5) = val5;
output(base_m + threadIdx.y + 8 * threadIdx.x, base_n + threadIdx.z + 8 * 6) = val6; output(base_m + thread_y + 8 * thread_x, base_n + thread_z + 8 * 6) = val6;
output(base_m + threadIdx.y + 8 * threadIdx.x, base_n + threadIdx.z + 8 * 7) = val7; output(base_m + thread_y + 8 * thread_x, base_n + thread_z + 8 * 7) = val7;
} else { } else {
#pragma unroll 7 #pragma unroll 7
for (int j = 0; j < max_j_write; j++) { for (int j = 0; j < max_j_write; j++) {
Scalar val = lhs_shmem[threadIdx.x + 8 * threadIdx.y + 64 * threadIdx.z + 512 * j]; Scalar val = lhs_shmem[thread_x + 8 * thread_y + 64 * thread_z + 512 * j];
output(base_m + threadIdx.y + 8 * threadIdx.x, base_n + threadIdx.z + 8 * j) = val; output(base_m + thread_y + 8 * thread_x, base_n + thread_z + 8 * j) = val;
} }
} }
} }
@@ -539,6 +542,8 @@ __device__ __forceinline__ void EigenFloatContractionKernelInternal16x16(const L
float4 lhs_pf0, rhs_pf0; float4 lhs_pf0, rhs_pf0;
float4 results[4]; float4 results[4];
const Index thread_x = threadIdx.x;
const Index thread_y = threadIdx.y;
for (int i = 0; i < 4; i++) { for (int i = 0; i < 4; i++) {
results[i].x = results[i].y = results[i].z = results[i].w = 0; results[i].x = results[i].y = results[i].z = results[i].w = 0;
} }
@@ -565,17 +570,17 @@ __device__ __forceinline__ void EigenFloatContractionKernelInternal16x16(const L
} \ } \
} }
Index lhs_vert = base_m + threadIdx.x * 4; Index lhs_vert = base_m + thread_x * 4;
for (Index k = 0; k < k_size; k += 16) { for (Index k = 0; k < k_size; k += 16) {
lhs_pf0 = internal::pset1<float4>(0); lhs_pf0 = internal::pset1<float4>(0);
rhs_pf0 = internal::pset1<float4>(0); rhs_pf0 = internal::pset1<float4>(0);
Index lhs_horiz = threadIdx.y + k; Index lhs_horiz = thread_y + k;
prefetch_lhs(lhs_pf0, lhs_vert, lhs_horiz) prefetch_lhs(lhs_pf0, lhs_vert, lhs_horiz)
Index rhs_vert = k + (threadIdx.x % 4) * 4; Index rhs_vert = k + (thread_x % 4) * 4;
Index rhs_horiz0 = (threadIdx.x >> 2) + threadIdx.y * 4 + base_n; Index rhs_horiz0 = (thread_x >> 2) + thread_y * 4 + base_n;
if (!CHECK_RHS_BOUNDARY) { if (!CHECK_RHS_BOUNDARY) {
if ((rhs_vert + 3) < k_size) { if ((rhs_vert + 3) < k_size) {
@@ -610,7 +615,7 @@ __device__ __forceinline__ void EigenFloatContractionKernelInternal16x16(const L
} }
float x1, x2; float x1, x2;
// TODO: The following can be a bitwise operation. // TODO: The following can be a bitwise operation.
if ((threadIdx.x % 8) < 4) { if ((thread_x % 8) < 4) {
x1 = rhs_pf0.y; x1 = rhs_pf0.y;
x2 = rhs_pf0.w; x2 = rhs_pf0.w;
} else { } else {
@@ -624,7 +629,7 @@ __device__ __forceinline__ void EigenFloatContractionKernelInternal16x16(const L
x1 = __shfl_xor_sync(0xFFFFFFFF, x1, 4); x1 = __shfl_xor_sync(0xFFFFFFFF, x1, 4);
x2 = __shfl_xor_sync(0xFFFFFFFF, x2, 4); x2 = __shfl_xor_sync(0xFFFFFFFF, x2, 4);
#endif #endif
if ((threadIdx.x % 8) < 4) { if ((thread_x % 8) < 4) {
rhs_pf0.y = x1; rhs_pf0.y = x1;
rhs_pf0.w = x2; rhs_pf0.w = x2;
} else { } else {
@@ -639,8 +644,8 @@ __device__ __forceinline__ void EigenFloatContractionKernelInternal16x16(const L
// Row 31 -> times (0, 4, 8, 12, 1, 5, 9, 13) for features 62, 63 // Row 31 -> times (0, 4, 8, 12, 1, 5, 9, 13) for features 62, 63
// Row 32 -> times (2, 6, 10, 14, 3, 7, 11, 15) for features 0, 1 // Row 32 -> times (2, 6, 10, 14, 3, 7, 11, 15) for features 0, 1
// ... // ...
rhs_shmem2[(threadIdx.x >> 3) + threadIdx.y * 2][threadIdx.x % 8] = make_float2(rhs_pf0.x, rhs_pf0.y); rhs_shmem2[(thread_x >> 3) + thread_y * 2][thread_x % 8] = make_float2(rhs_pf0.x, rhs_pf0.y);
rhs_shmem2[(threadIdx.x >> 3) + threadIdx.y * 2 + 32][threadIdx.x % 8] = make_float2(rhs_pf0.z, rhs_pf0.w); rhs_shmem2[(thread_x >> 3) + thread_y * 2 + 32][thread_x % 8] = make_float2(rhs_pf0.z, rhs_pf0.w);
// Row 0 (time 0) -> features (0, 1), (4, 5), .. (28, 29), (32, 33), .. (60, 61) // Row 0 (time 0) -> features (0, 1), (4, 5), .. (28, 29), (32, 33), .. (60, 61)
// Row 1 (time 1) -> features (0, 1), (4, 5), .. (28, 29), (32, 33), .. (60, 61) // Row 1 (time 1) -> features (0, 1), (4, 5), .. (28, 29), (32, 33), .. (60, 61)
@@ -649,8 +654,8 @@ __device__ __forceinline__ void EigenFloatContractionKernelInternal16x16(const L
// Row 16 (time 0) -> features (2, 3), (6, 7), .. (30, 31), (34, 35), .. (62, 63) // Row 16 (time 0) -> features (2, 3), (6, 7), .. (30, 31), (34, 35), .. (62, 63)
// ... // ...
lhs_shmem2[threadIdx.y][threadIdx.x] = make_float2(lhs_pf0.x, lhs_pf0.y); lhs_shmem2[thread_y][thread_x] = make_float2(lhs_pf0.x, lhs_pf0.y);
lhs_shmem2[threadIdx.y + 16][threadIdx.x] = make_float2(lhs_pf0.z, lhs_pf0.w); lhs_shmem2[thread_y + 16][thread_x] = make_float2(lhs_pf0.z, lhs_pf0.w);
#define add_vals(fl1, fl2, fr1, fr2) \ #define add_vals(fl1, fl2, fr1, fr2) \
results[0].x += fl1.x * fr1.x; \ results[0].x += fl1.x * fr1.x; \
@@ -679,10 +684,10 @@ __device__ __forceinline__ void EigenFloatContractionKernelInternal16x16(const L
#pragma unroll #pragma unroll
for (int koff = 0; koff < 16; koff++) { for (int koff = 0; koff < 16; koff++) {
// 32 x threads. // 32 x threads.
float2 fl1 = lhs_shmem2[koff][threadIdx.x]; float2 fl1 = lhs_shmem2[koff][thread_x];
float2 fl2 = lhs_shmem2[koff + 16][threadIdx.x]; float2 fl2 = lhs_shmem2[koff + 16][thread_x];
int start_feature = threadIdx.y * 4; int start_feature = thread_y * 4;
float2 fr1 = rhs_shmem2[(start_feature >> 1) + 32 * ((koff % 4) / 2)][koff / 4 + (koff % 2) * 4]; float2 fr1 = rhs_shmem2[(start_feature >> 1) + 32 * ((koff % 4) / 2)][koff / 4 + (koff % 2) * 4];
float2 fr2 = rhs_shmem2[(start_feature >> 1) + 1 + 32 * ((koff % 4) / 2)][koff / 4 + (koff % 2) * 4]; float2 fr2 = rhs_shmem2[(start_feature >> 1) + 1 + 32 * ((koff % 4) / 2)][koff / 4 + (koff % 2) * 4];
@@ -694,7 +699,7 @@ __device__ __forceinline__ void EigenFloatContractionKernelInternal16x16(const L
#undef prefetch_lhs #undef prefetch_lhs
#undef add_vals #undef add_vals
Index horiz_base = threadIdx.y * 4 + base_n; Index horiz_base = thread_y * 4 + base_n;
if (!CHECK_LHS_BOUNDARY && !CHECK_RHS_BOUNDARY) { if (!CHECK_LHS_BOUNDARY && !CHECK_RHS_BOUNDARY) {
for (int i = 0; i < 4; i++) { for (int i = 0; i < 4; i++) {
output(lhs_vert, horiz_base + i) = results[i].x; output(lhs_vert, horiz_base + i) = results[i].x;
@@ -770,11 +775,15 @@ __device__ __forceinline__ void EigenFloatContractionKernelInternal(const LhsMap
float4 rhs_pf0, rhs_pf1; float4 rhs_pf0, rhs_pf1;
float4 results[8]; float4 results[8];
const Index thread_x = threadIdx.x;
const Index thread_y = threadIdx.y;
for (int i = 0; i < 8; i++) { for (int i = 0; i < 8; i++) {
results[i].x = results[i].y = results[i].z = results[i].w = 0; results[i].x = results[i].y = results[i].z = results[i].w = 0;
} }
Index lhs_vert = base_m + threadIdx.x * 4 + (threadIdx.y % 4) * 32; Index lhs_vert = base_m + thread_x * 4 + (thread_y % 4) * 32;
for (Index k = 0; k < k_size; k += 32) { for (Index k = 0; k < k_size; k += 32) {
lhs_pf0 = internal::pset1<float4>(0); lhs_pf0 = internal::pset1<float4>(0);
lhs_pf1 = internal::pset1<float4>(0); lhs_pf1 = internal::pset1<float4>(0);
@@ -785,123 +794,123 @@ __device__ __forceinline__ void EigenFloatContractionKernelInternal(const LhsMap
rhs_pf1 = internal::pset1<float4>(0); rhs_pf1 = internal::pset1<float4>(0);
if (!CHECK_LHS_BOUNDARY) { if (!CHECK_LHS_BOUNDARY) {
if ((threadIdx.y / 4 + k + 24) < k_size) { if ((thread_y / 4 + k + 24) < k_size) {
lhs_pf0 = lhs.template loadPacket<float4, Unaligned>(lhs_vert, (threadIdx.y / 4 + k)); lhs_pf0 = lhs.template loadPacket<float4, Unaligned>(lhs_vert, (thread_y / 4 + k));
lhs_pf1 = lhs.template loadPacket<float4, Unaligned>(lhs_vert, (threadIdx.y / 4 + k + 8)); lhs_pf1 = lhs.template loadPacket<float4, Unaligned>(lhs_vert, (thread_y / 4 + k + 8));
lhs_pf2 = lhs.template loadPacket<float4, Unaligned>(lhs_vert, (threadIdx.y / 4 + k + 16)); lhs_pf2 = lhs.template loadPacket<float4, Unaligned>(lhs_vert, (thread_y / 4 + k + 16));
lhs_pf3 = lhs.template loadPacket<float4, Unaligned>(lhs_vert, (threadIdx.y / 4 + k + 24)); lhs_pf3 = lhs.template loadPacket<float4, Unaligned>(lhs_vert, (thread_y / 4 + k + 24));
} else if ((threadIdx.y / 4 + k + 16) < k_size) { } else if ((thread_y / 4 + k + 16) < k_size) {
lhs_pf0 = lhs.template loadPacket<float4, Unaligned>(lhs_vert, (threadIdx.y / 4 + k)); lhs_pf0 = lhs.template loadPacket<float4, Unaligned>(lhs_vert, (thread_y / 4 + k));
lhs_pf1 = lhs.template loadPacket<float4, Unaligned>(lhs_vert, (threadIdx.y / 4 + k + 8)); lhs_pf1 = lhs.template loadPacket<float4, Unaligned>(lhs_vert, (thread_y / 4 + k + 8));
lhs_pf2 = lhs.template loadPacket<float4, Unaligned>(lhs_vert, (threadIdx.y / 4 + k + 16)); lhs_pf2 = lhs.template loadPacket<float4, Unaligned>(lhs_vert, (thread_y / 4 + k + 16));
} else if ((threadIdx.y / 4 + k + 8) < k_size) { } else if ((thread_y / 4 + k + 8) < k_size) {
lhs_pf0 = lhs.template loadPacket<float4, Unaligned>(lhs_vert, (threadIdx.y / 4 + k)); lhs_pf0 = lhs.template loadPacket<float4, Unaligned>(lhs_vert, (thread_y / 4 + k));
lhs_pf1 = lhs.template loadPacket<float4, Unaligned>(lhs_vert, (threadIdx.y / 4 + k + 8)); lhs_pf1 = lhs.template loadPacket<float4, Unaligned>(lhs_vert, (thread_y / 4 + k + 8));
} else if ((threadIdx.y / 4 + k) < k_size) { } else if ((thread_y / 4 + k) < k_size) {
lhs_pf0 = lhs.template loadPacket<float4, Unaligned>(lhs_vert, (threadIdx.y / 4 + k)); lhs_pf0 = lhs.template loadPacket<float4, Unaligned>(lhs_vert, (thread_y / 4 + k));
} }
} else { } else {
// just CHECK_LHS_BOUNDARY // just CHECK_LHS_BOUNDARY
if (lhs_vert + 3 < m_size) { if (lhs_vert + 3 < m_size) {
if ((threadIdx.y / 4 + k + 24) < k_size) { if ((thread_y / 4 + k + 24) < k_size) {
lhs_pf0 = lhs.template loadPacket<float4, Unaligned>(lhs_vert, (threadIdx.y / 4 + k)); lhs_pf0 = lhs.template loadPacket<float4, Unaligned>(lhs_vert, (thread_y / 4 + k));
lhs_pf1 = lhs.template loadPacket<float4, Unaligned>(lhs_vert, (threadIdx.y / 4 + k + 8)); lhs_pf1 = lhs.template loadPacket<float4, Unaligned>(lhs_vert, (thread_y / 4 + k + 8));
lhs_pf2 = lhs.template loadPacket<float4, Unaligned>(lhs_vert, (threadIdx.y / 4 + k + 16)); lhs_pf2 = lhs.template loadPacket<float4, Unaligned>(lhs_vert, (thread_y / 4 + k + 16));
lhs_pf3 = lhs.template loadPacket<float4, Unaligned>(lhs_vert, (threadIdx.y / 4 + k + 24)); lhs_pf3 = lhs.template loadPacket<float4, Unaligned>(lhs_vert, (thread_y / 4 + k + 24));
} else if ((threadIdx.y / 4 + k + 16) < k_size) { } else if ((thread_y / 4 + k + 16) < k_size) {
lhs_pf0 = lhs.template loadPacket<float4, Unaligned>(lhs_vert, (threadIdx.y / 4 + k)); lhs_pf0 = lhs.template loadPacket<float4, Unaligned>(lhs_vert, (thread_y / 4 + k));
lhs_pf1 = lhs.template loadPacket<float4, Unaligned>(lhs_vert, (threadIdx.y / 4 + k + 8)); lhs_pf1 = lhs.template loadPacket<float4, Unaligned>(lhs_vert, (thread_y / 4 + k + 8));
lhs_pf2 = lhs.template loadPacket<float4, Unaligned>(lhs_vert, (threadIdx.y / 4 + k + 16)); lhs_pf2 = lhs.template loadPacket<float4, Unaligned>(lhs_vert, (thread_y / 4 + k + 16));
} else if ((threadIdx.y / 4 + k + 8) < k_size) { } else if ((thread_y / 4 + k + 8) < k_size) {
lhs_pf0 = lhs.template loadPacket<float4, Unaligned>(lhs_vert, (threadIdx.y / 4 + k)); lhs_pf0 = lhs.template loadPacket<float4, Unaligned>(lhs_vert, (thread_y / 4 + k));
lhs_pf1 = lhs.template loadPacket<float4, Unaligned>(lhs_vert, (threadIdx.y / 4 + k + 8)); lhs_pf1 = lhs.template loadPacket<float4, Unaligned>(lhs_vert, (thread_y / 4 + k + 8));
} else if ((threadIdx.y / 4 + k) < k_size) { } else if ((thread_y / 4 + k) < k_size) {
lhs_pf0 = lhs.template loadPacket<float4, Unaligned>(lhs_vert, (threadIdx.y / 4 + k)); lhs_pf0 = lhs.template loadPacket<float4, Unaligned>(lhs_vert, (thread_y / 4 + k));
} }
} else if (lhs_vert + 2 < m_size) { } else if (lhs_vert + 2 < m_size) {
if ((threadIdx.y / 4 + k + 24) < k_size) { if ((thread_y / 4 + k + 24) < k_size) {
lhs_pf0.x = lhs(lhs_vert + 0, (threadIdx.y / 4 + k)); lhs_pf0.x = lhs(lhs_vert + 0, (thread_y / 4 + k));
lhs_pf0.y = lhs(lhs_vert + 1, (threadIdx.y / 4 + k)); lhs_pf0.y = lhs(lhs_vert + 1, (thread_y / 4 + k));
lhs_pf0.z = lhs(lhs_vert + 2, (threadIdx.y / 4 + k)); lhs_pf0.z = lhs(lhs_vert + 2, (thread_y / 4 + k));
lhs_pf1.x = lhs(lhs_vert + 0, (threadIdx.y / 4 + k + 8)); lhs_pf1.x = lhs(lhs_vert + 0, (thread_y / 4 + k + 8));
lhs_pf1.y = lhs(lhs_vert + 1, (threadIdx.y / 4 + k + 8)); lhs_pf1.y = lhs(lhs_vert + 1, (thread_y / 4 + k + 8));
lhs_pf1.z = lhs(lhs_vert + 2, (threadIdx.y / 4 + k + 8)); lhs_pf1.z = lhs(lhs_vert + 2, (thread_y / 4 + k + 8));
lhs_pf2.x = lhs(lhs_vert + 0, (threadIdx.y / 4 + k + 16)); lhs_pf2.x = lhs(lhs_vert + 0, (thread_y / 4 + k + 16));
lhs_pf2.y = lhs(lhs_vert + 1, (threadIdx.y / 4 + k + 16)); lhs_pf2.y = lhs(lhs_vert + 1, (thread_y / 4 + k + 16));
lhs_pf2.z = lhs(lhs_vert + 2, (threadIdx.y / 4 + k + 16)); lhs_pf2.z = lhs(lhs_vert + 2, (thread_y / 4 + k + 16));
lhs_pf3.x = lhs(lhs_vert + 0, (threadIdx.y / 4 + k + 24)); lhs_pf3.x = lhs(lhs_vert + 0, (thread_y / 4 + k + 24));
lhs_pf3.y = lhs(lhs_vert + 1, (threadIdx.y / 4 + k + 24)); lhs_pf3.y = lhs(lhs_vert + 1, (thread_y / 4 + k + 24));
lhs_pf3.z = lhs(lhs_vert + 2, (threadIdx.y / 4 + k + 24)); lhs_pf3.z = lhs(lhs_vert + 2, (thread_y / 4 + k + 24));
} else if ((threadIdx.y / 4 + k + 16) < k_size) { } else if ((thread_y / 4 + k + 16) < k_size) {
lhs_pf0.x = lhs(lhs_vert + 0, (threadIdx.y / 4 + k)); lhs_pf0.x = lhs(lhs_vert + 0, (thread_y / 4 + k));
lhs_pf0.y = lhs(lhs_vert + 1, (threadIdx.y / 4 + k)); lhs_pf0.y = lhs(lhs_vert + 1, (thread_y / 4 + k));
lhs_pf0.z = lhs(lhs_vert + 2, (threadIdx.y / 4 + k)); lhs_pf0.z = lhs(lhs_vert + 2, (thread_y / 4 + k));
lhs_pf1.x = lhs(lhs_vert + 0, (threadIdx.y / 4 + k + 8)); lhs_pf1.x = lhs(lhs_vert + 0, (thread_y / 4 + k + 8));
lhs_pf1.y = lhs(lhs_vert + 1, (threadIdx.y / 4 + k + 8)); lhs_pf1.y = lhs(lhs_vert + 1, (thread_y / 4 + k + 8));
lhs_pf1.z = lhs(lhs_vert + 2, (threadIdx.y / 4 + k + 8)); lhs_pf1.z = lhs(lhs_vert + 2, (thread_y / 4 + k + 8));
lhs_pf2.x = lhs(lhs_vert + 0, (threadIdx.y / 4 + k + 16)); lhs_pf2.x = lhs(lhs_vert + 0, (thread_y / 4 + k + 16));
lhs_pf2.y = lhs(lhs_vert + 1, (threadIdx.y / 4 + k + 16)); lhs_pf2.y = lhs(lhs_vert + 1, (thread_y / 4 + k + 16));
lhs_pf2.z = lhs(lhs_vert + 2, (threadIdx.y / 4 + k + 16)); lhs_pf2.z = lhs(lhs_vert + 2, (thread_y / 4 + k + 16));
} else if ((threadIdx.y / 4 + k + 8) < k_size) { } else if ((thread_y / 4 + k + 8) < k_size) {
lhs_pf0.x = lhs(lhs_vert + 0, (threadIdx.y / 4 + k)); lhs_pf0.x = lhs(lhs_vert + 0, (thread_y / 4 + k));
lhs_pf0.y = lhs(lhs_vert + 1, (threadIdx.y / 4 + k)); lhs_pf0.y = lhs(lhs_vert + 1, (thread_y / 4 + k));
lhs_pf0.z = lhs(lhs_vert + 2, (threadIdx.y / 4 + k)); lhs_pf0.z = lhs(lhs_vert + 2, (thread_y / 4 + k));
lhs_pf1.x = lhs(lhs_vert + 0, (threadIdx.y / 4 + k + 8)); lhs_pf1.x = lhs(lhs_vert + 0, (thread_y / 4 + k + 8));
lhs_pf1.y = lhs(lhs_vert + 1, (threadIdx.y / 4 + k + 8)); lhs_pf1.y = lhs(lhs_vert + 1, (thread_y / 4 + k + 8));
lhs_pf1.z = lhs(lhs_vert + 2, (threadIdx.y / 4 + k + 8)); lhs_pf1.z = lhs(lhs_vert + 2, (thread_y / 4 + k + 8));
} else if ((threadIdx.y / 4 + k) < k_size) { } else if ((thread_y / 4 + k) < k_size) {
lhs_pf0.x = lhs(lhs_vert + 0, (threadIdx.y / 4 + k)); lhs_pf0.x = lhs(lhs_vert + 0, (thread_y / 4 + k));
lhs_pf0.y = lhs(lhs_vert + 1, (threadIdx.y / 4 + k)); lhs_pf0.y = lhs(lhs_vert + 1, (thread_y / 4 + k));
lhs_pf0.z = lhs(lhs_vert + 2, (threadIdx.y / 4 + k)); lhs_pf0.z = lhs(lhs_vert + 2, (thread_y / 4 + k));
} }
} else if (lhs_vert + 1 < m_size) { } else if (lhs_vert + 1 < m_size) {
if ((threadIdx.y / 4 + k + 24) < k_size) { if ((thread_y / 4 + k + 24) < k_size) {
lhs_pf0.x = lhs(lhs_vert + 0, (threadIdx.y / 4 + k)); lhs_pf0.x = lhs(lhs_vert + 0, (thread_y / 4 + k));
lhs_pf0.y = lhs(lhs_vert + 1, (threadIdx.y / 4 + k)); lhs_pf0.y = lhs(lhs_vert + 1, (thread_y / 4 + k));
lhs_pf1.x = lhs(lhs_vert + 0, (threadIdx.y / 4 + k + 8)); lhs_pf1.x = lhs(lhs_vert + 0, (thread_y / 4 + k + 8));
lhs_pf1.y = lhs(lhs_vert + 1, (threadIdx.y / 4 + k + 8)); lhs_pf1.y = lhs(lhs_vert + 1, (thread_y / 4 + k + 8));
lhs_pf2.x = lhs(lhs_vert + 0, (threadIdx.y / 4 + k + 16)); lhs_pf2.x = lhs(lhs_vert + 0, (thread_y / 4 + k + 16));
lhs_pf2.y = lhs(lhs_vert + 1, (threadIdx.y / 4 + k + 16)); lhs_pf2.y = lhs(lhs_vert + 1, (thread_y / 4 + k + 16));
lhs_pf3.x = lhs(lhs_vert + 0, (threadIdx.y / 4 + k + 24)); lhs_pf3.x = lhs(lhs_vert + 0, (thread_y / 4 + k + 24));
lhs_pf3.y = lhs(lhs_vert + 1, (threadIdx.y / 4 + k + 24)); lhs_pf3.y = lhs(lhs_vert + 1, (thread_y / 4 + k + 24));
} else if ((threadIdx.y / 4 + k + 16) < k_size) { } else if ((thread_y / 4 + k + 16) < k_size) {
lhs_pf0.x = lhs(lhs_vert + 0, (threadIdx.y / 4 + k)); lhs_pf0.x = lhs(lhs_vert + 0, (thread_y / 4 + k));
lhs_pf0.y = lhs(lhs_vert + 1, (threadIdx.y / 4 + k)); lhs_pf0.y = lhs(lhs_vert + 1, (thread_y / 4 + k));
lhs_pf1.x = lhs(lhs_vert + 0, (threadIdx.y / 4 + k + 8)); lhs_pf1.x = lhs(lhs_vert + 0, (thread_y / 4 + k + 8));
lhs_pf1.y = lhs(lhs_vert + 1, (threadIdx.y / 4 + k + 8)); lhs_pf1.y = lhs(lhs_vert + 1, (thread_y / 4 + k + 8));
lhs_pf2.x = lhs(lhs_vert + 0, (threadIdx.y / 4 + k + 16)); lhs_pf2.x = lhs(lhs_vert + 0, (thread_y / 4 + k + 16));
lhs_pf2.y = lhs(lhs_vert + 1, (threadIdx.y / 4 + k + 16)); lhs_pf2.y = lhs(lhs_vert + 1, (thread_y / 4 + k + 16));
} else if ((threadIdx.y / 4 + k + 8) < k_size) { } else if ((thread_y / 4 + k + 8) < k_size) {
lhs_pf0.x = lhs(lhs_vert + 0, (threadIdx.y / 4 + k)); lhs_pf0.x = lhs(lhs_vert + 0, (thread_y / 4 + k));
lhs_pf0.y = lhs(lhs_vert + 1, (threadIdx.y / 4 + k)); lhs_pf0.y = lhs(lhs_vert + 1, (thread_y / 4 + k));
lhs_pf1.x = lhs(lhs_vert + 0, (threadIdx.y / 4 + k + 8)); lhs_pf1.x = lhs(lhs_vert + 0, (thread_y / 4 + k + 8));
lhs_pf1.y = lhs(lhs_vert + 1, (threadIdx.y / 4 + k + 8)); lhs_pf1.y = lhs(lhs_vert + 1, (thread_y / 4 + k + 8));
} else if ((threadIdx.y / 4 + k) < k_size) { } else if ((thread_y / 4 + k) < k_size) {
lhs_pf0.x = lhs(lhs_vert + 0, (threadIdx.y / 4 + k)); lhs_pf0.x = lhs(lhs_vert + 0, (thread_y / 4 + k));
lhs_pf0.y = lhs(lhs_vert + 1, (threadIdx.y / 4 + k)); lhs_pf0.y = lhs(lhs_vert + 1, (thread_y / 4 + k));
} }
} else if (lhs_vert < m_size) { } else if (lhs_vert < m_size) {
if ((threadIdx.y / 4 + k + 24) < k_size) { if ((thread_y / 4 + k + 24) < k_size) {
lhs_pf0.x = lhs(lhs_vert + 0, (threadIdx.y / 4 + k)); lhs_pf0.x = lhs(lhs_vert + 0, (thread_y / 4 + k));
lhs_pf1.x = lhs(lhs_vert + 0, (threadIdx.y / 4 + k + 8)); lhs_pf1.x = lhs(lhs_vert + 0, (thread_y / 4 + k + 8));
lhs_pf2.x = lhs(lhs_vert + 0, (threadIdx.y / 4 + k + 16)); lhs_pf2.x = lhs(lhs_vert + 0, (thread_y / 4 + k + 16));
lhs_pf3.x = lhs(lhs_vert + 0, (threadIdx.y / 4 + k + 24)); lhs_pf3.x = lhs(lhs_vert + 0, (thread_y / 4 + k + 24));
} else if ((threadIdx.y / 4 + k + 16) < k_size) { } else if ((thread_y / 4 + k + 16) < k_size) {
lhs_pf0.x = lhs(lhs_vert + 0, (threadIdx.y / 4 + k)); lhs_pf0.x = lhs(lhs_vert + 0, (thread_y / 4 + k));
lhs_pf1.x = lhs(lhs_vert + 0, (threadIdx.y / 4 + k + 8)); lhs_pf1.x = lhs(lhs_vert + 0, (thread_y / 4 + k + 8));
lhs_pf2.x = lhs(lhs_vert + 0, (threadIdx.y / 4 + k + 16)); lhs_pf2.x = lhs(lhs_vert + 0, (thread_y / 4 + k + 16));
} else if ((threadIdx.y / 4 + k + 8) < k_size) { } else if ((thread_y / 4 + k + 8) < k_size) {
lhs_pf0.x = lhs(lhs_vert + 0, (threadIdx.y / 4 + k)); lhs_pf0.x = lhs(lhs_vert + 0, (thread_y / 4 + k));
lhs_pf1.x = lhs(lhs_vert + 0, (threadIdx.y / 4 + k + 8)); lhs_pf1.x = lhs(lhs_vert + 0, (thread_y / 4 + k + 8));
} else if ((threadIdx.y / 4 + k) < k_size) { } else if ((thread_y / 4 + k) < k_size) {
lhs_pf0.x = lhs(lhs_vert + 0, (threadIdx.y / 4 + k)); lhs_pf0.x = lhs(lhs_vert + 0, (thread_y / 4 + k));
} }
} }
} }
__syncthreads(); __syncthreads();
Index rhs_vert = k + threadIdx.x * 4; Index rhs_vert = k + thread_x * 4;
Index rhs_horiz0 = threadIdx.y * 2 + base_n; Index rhs_horiz0 = thread_y * 2 + base_n;
Index rhs_horiz1 = threadIdx.y * 2 + 1 + base_n; Index rhs_horiz1 = thread_y * 2 + 1 + base_n;
if (!CHECK_RHS_BOUNDARY) { if (!CHECK_RHS_BOUNDARY) {
if ((rhs_vert + 3) < k_size) { if ((rhs_vert + 3) < k_size) {
// just CHECK_RHS_BOUNDARY // just CHECK_RHS_BOUNDARY
@@ -938,12 +947,12 @@ __device__ __forceinline__ void EigenFloatContractionKernelInternal(const LhsMap
rhs_pf1.x = rhs(rhs_vert, rhs_horiz1); rhs_pf1.x = rhs(rhs_vert, rhs_horiz1);
rhs_pf1.y = rhs(rhs_vert + 1, rhs_horiz1); rhs_pf1.y = rhs(rhs_vert + 1, rhs_horiz1);
rhs_pf1.z = rhs(rhs_vert + 2, rhs_horiz1); rhs_pf1.z = rhs(rhs_vert + 2, rhs_horiz1);
} else if (k + threadIdx.x * 4 + 1 < k_size) { } else if (k + thread_x * 4 + 1 < k_size) {
rhs_pf0.x = rhs(rhs_vert, rhs_horiz0); rhs_pf0.x = rhs(rhs_vert, rhs_horiz0);
rhs_pf0.y = rhs(rhs_vert + 1, rhs_horiz0); rhs_pf0.y = rhs(rhs_vert + 1, rhs_horiz0);
rhs_pf1.x = rhs(rhs_vert, rhs_horiz1); rhs_pf1.x = rhs(rhs_vert, rhs_horiz1);
rhs_pf1.y = rhs(rhs_vert + 1, rhs_horiz1); rhs_pf1.y = rhs(rhs_vert + 1, rhs_horiz1);
} else if (k + threadIdx.x * 4 < k_size) { } else if (k + thread_x * 4 < k_size) {
rhs_pf0.x = rhs(rhs_vert, rhs_horiz0); rhs_pf0.x = rhs(rhs_vert, rhs_horiz0);
rhs_pf1.x = rhs(rhs_vert, rhs_horiz1); rhs_pf1.x = rhs(rhs_vert, rhs_horiz1);
} }
@@ -970,17 +979,17 @@ __device__ __forceinline__ void EigenFloatContractionKernelInternal(const LhsMap
// Row 1 -> times (0, 4, 8, .. 28) for features 2, 3. // Row 1 -> times (0, 4, 8, .. 28) for features 2, 3.
// .. // ..
// Row 31 -> times (0, 4, 8, .. 28) for features 62, 63 // Row 31 -> times (0, 4, 8, .. 28) for features 62, 63
rhs_shmem2[threadIdx.y][threadIdx.x] = make_float2(rhs_pf0.x, rhs_pf1.x); rhs_shmem2[thread_y][thread_x] = make_float2(rhs_pf0.x, rhs_pf1.x);
// Row 32 -> times (1, 5, 9, .. 29) for features 0, 1. // Row 32 -> times (1, 5, 9, .. 29) for features 0, 1.
// Row 33 -> times (1, 5, 9, .. 29) for features 2, 3. // Row 33 -> times (1, 5, 9, .. 29) for features 2, 3.
// .. // ..
rhs_shmem2[threadIdx.y + 32][threadIdx.x] = make_float2(rhs_pf0.y, rhs_pf1.y); rhs_shmem2[thread_y + 32][thread_x] = make_float2(rhs_pf0.y, rhs_pf1.y);
// Row 64 -> times (2, 6, 10, .. 30) for features 0, 1. // Row 64 -> times (2, 6, 10, .. 30) for features 0, 1.
// Row 65 -> times (2, 6, 10, .. 30) for features 2, 3. // Row 65 -> times (2, 6, 10, .. 30) for features 2, 3.
rhs_shmem2[threadIdx.y + 64][threadIdx.x] = make_float2(rhs_pf0.z, rhs_pf1.z); rhs_shmem2[thread_y + 64][thread_x] = make_float2(rhs_pf0.z, rhs_pf1.z);
// Row 96 -> times (3, 7, 11, .. 31) for features 0, 1. // Row 96 -> times (3, 7, 11, .. 31) for features 0, 1.
// Row 97 -> times (3, 7, 11, .. 31) for features 2, 3. // Row 97 -> times (3, 7, 11, .. 31) for features 2, 3.
rhs_shmem2[threadIdx.y + 96][threadIdx.x] = make_float2(rhs_pf0.w, rhs_pf1.w); rhs_shmem2[thread_y + 96][thread_x] = make_float2(rhs_pf0.w, rhs_pf1.w);
// LHS. // LHS.
// Row 0 (time 0) -> features (0, 1), (4, 5), .. (28, 29), (32, 33), .. (60, 61) .. (124, 125) // Row 0 (time 0) -> features (0, 1), (4, 5), .. (28, 29), (32, 33), .. (60, 61) .. (124, 125)
@@ -1026,26 +1035,26 @@ __device__ __forceinline__ void EigenFloatContractionKernelInternal(const LhsMap
results[6].w += a_feat2.y * f4.x; \ results[6].w += a_feat2.y * f4.x; \
results[7].w += a_feat2.y * f4.y; results[7].w += a_feat2.y * f4.y;
lhs_shmem2[threadIdx.y / 4][threadIdx.x + (threadIdx.y % 4) * 8] = make_float2(lhs_pf0.x, lhs_pf0.y); lhs_shmem2[thread_y / 4][thread_x + (thread_y % 4) * 8] = make_float2(lhs_pf0.x, lhs_pf0.y);
lhs_shmem2[threadIdx.y / 4 + 8][threadIdx.x + (threadIdx.y % 4) * 8] = make_float2(lhs_pf1.x, lhs_pf1.y); lhs_shmem2[thread_y / 4 + 8][thread_x + (thread_y % 4) * 8] = make_float2(lhs_pf1.x, lhs_pf1.y);
lhs_shmem2[threadIdx.y / 4 + 16][threadIdx.x + (threadIdx.y % 4) * 8] = make_float2(lhs_pf2.x, lhs_pf2.y); lhs_shmem2[thread_y / 4 + 16][thread_x + (thread_y % 4) * 8] = make_float2(lhs_pf2.x, lhs_pf2.y);
lhs_shmem2[threadIdx.y / 4 + 24][threadIdx.x + (threadIdx.y % 4) * 8] = make_float2(lhs_pf3.x, lhs_pf3.y); lhs_shmem2[thread_y / 4 + 24][thread_x + (thread_y % 4) * 8] = make_float2(lhs_pf3.x, lhs_pf3.y);
lhs_shmem2[threadIdx.y / 4 + 32][threadIdx.x + (threadIdx.y % 4) * 8] = make_float2(lhs_pf0.z, lhs_pf0.w); lhs_shmem2[thread_y / 4 + 32][thread_x + (thread_y % 4) * 8] = make_float2(lhs_pf0.z, lhs_pf0.w);
lhs_shmem2[threadIdx.y / 4 + 40][threadIdx.x + (threadIdx.y % 4) * 8] = make_float2(lhs_pf1.z, lhs_pf1.w); lhs_shmem2[thread_y / 4 + 40][thread_x + (thread_y % 4) * 8] = make_float2(lhs_pf1.z, lhs_pf1.w);
lhs_shmem2[threadIdx.y / 4 + 48][threadIdx.x + (threadIdx.y % 4) * 8] = make_float2(lhs_pf2.z, lhs_pf2.w); lhs_shmem2[thread_y / 4 + 48][thread_x + (thread_y % 4) * 8] = make_float2(lhs_pf2.z, lhs_pf2.w);
lhs_shmem2[threadIdx.y / 4 + 56][threadIdx.x + (threadIdx.y % 4) * 8] = make_float2(lhs_pf3.z, lhs_pf3.w); lhs_shmem2[thread_y / 4 + 56][thread_x + (thread_y % 4) * 8] = make_float2(lhs_pf3.z, lhs_pf3.w);
__syncthreads(); __syncthreads();
// Do the multiplies. // Do the multiplies.
#pragma unroll #pragma unroll
for (int koff = 0; koff < 32; koff++) { for (int koff = 0; koff < 32; koff++) {
float2 a3 = lhs_shmem2[koff][threadIdx.x + (threadIdx.y % 4) * 8]; float2 a3 = lhs_shmem2[koff][thread_x + (thread_y % 4) * 8];
float2 a4 = lhs_shmem2[koff + 32][threadIdx.x + (threadIdx.y % 4) * 8]; float2 a4 = lhs_shmem2[koff + 32][thread_x + (thread_y % 4) * 8];
// first feature is at (threadIdx.y/4) * 8 last is at start + 8. // first feature is at (thread_y/4) * 8 last is at start + 8.
int start_feature = (threadIdx.y / 4) * 8; int start_feature = (thread_y / 4) * 8;
float2 br1 = rhs_shmem2[start_feature / 2 + (koff % 4) * 32][koff / 4]; float2 br1 = rhs_shmem2[start_feature / 2 + (koff % 4) * 32][koff / 4];
float2 br2 = rhs_shmem2[start_feature / 2 + 1 + (koff % 4) * 32][koff / 4]; float2 br2 = rhs_shmem2[start_feature / 2 + 1 + (koff % 4) * 32][koff / 4];
@@ -1060,7 +1069,7 @@ __device__ __forceinline__ void EigenFloatContractionKernelInternal(const LhsMap
#undef add_vals #undef add_vals
__syncthreads(); __syncthreads();
Index horiz_base = (threadIdx.y / 4) * 8 + base_n; Index horiz_base = (thread_y / 4) * 8 + base_n;
if (!CHECK_LHS_BOUNDARY && !CHECK_RHS_BOUNDARY) { if (!CHECK_LHS_BOUNDARY && !CHECK_RHS_BOUNDARY) {
for (int i = 0; i < 8; i++) { for (int i = 0; i < 8; i++) {
output(lhs_vert, horiz_base + i) = results[i].x; output(lhs_vert, horiz_base + i) = results[i].x;

24
unsupported/Eigen/src/Tensor/TensorConvolution.h
View File

@@ -666,8 +666,8 @@ template <typename InputEvaluator, typename Index, typename InputDims>
__global__ EIGEN_HIP_LAUNCH_BOUNDS_1024 void EigenConvolutionKernel3D( __global__ EIGEN_HIP_LAUNCH_BOUNDS_1024 void EigenConvolutionKernel3D(
InputEvaluator eval, const internal::IndexMapper<Index, InputDims, 3, InputEvaluator::Layout> indexMapper, InputEvaluator eval, const internal::IndexMapper<Index, InputDims, 3, InputEvaluator::Layout> indexMapper,
const float* __restrict kernel, const size_t numPlanes, const size_t numX, const size_t maxX, const size_t numY, const float* __restrict kernel, const size_t numPlanes, const size_t numX, const size_t maxX, const size_t numY,
const size_t maxY, const size_t numZ, const size_t maxZ, const size_t kernelSizeX, const size_t kernelSizeY, const size_t maxY, const size_t numZ, const size_t maxZ, const int kernelSizeX, const int kernelSizeY,
const size_t kernelSizeZ, float* buffer) { const int kernelSizeZ, float* buffer) {
#if defined(EIGEN_HIPCC) #if defined(EIGEN_HIPCC)
HIP_DYNAMIC_SHARED(float, s) HIP_DYNAMIC_SHARED(float, s)
#else #else
@@ -675,19 +675,25 @@ __global__ EIGEN_HIP_LAUNCH_BOUNDS_1024 void EigenConvolutionKernel3D(
#endif #endif
// Load inputs to shared memory // Load inputs to shared memory
const int first_x = blockIdx.x * maxX; const int first_x = blockIdx.x * static_cast<int>(maxX);
const int last_x = (first_x + maxX < numX ? first_x + maxX : numX) - 1; const int last_x = (first_x + static_cast<int>(maxX) < static_cast<int>(numX) ? first_x + static_cast<int>(maxX)
: static_cast<int>(numX)) -
1;
const int num_x_input = last_x - first_x + kernelSizeX; const int num_x_input = last_x - first_x + kernelSizeX;
const int first_y = blockIdx.y * maxY; const int first_y = blockIdx.y * static_cast<int>(maxY);
const int last_y = (first_y + maxY < numY ? first_y + maxY : numY) - 1; const int last_y = (first_y + static_cast<int>(maxY) < static_cast<int>(numY) ? first_y + static_cast<int>(maxY)
: static_cast<int>(numY)) -
1;
const int num_y_input = last_y - first_y + kernelSizeY; const int num_y_input = last_y - first_y + kernelSizeY;
const int first_z = blockIdx.z * maxZ; const int first_z = blockIdx.z * static_cast<int>(maxZ);
const int last_z = (first_z + maxZ < numZ ? first_z + maxZ : numZ) - 1; const int last_z = (first_z + static_cast<int>(maxZ) < static_cast<int>(numZ) ? first_z + static_cast<int>(maxZ)
: static_cast<int>(numZ)) -
1;
const int num_z_input = last_z - first_z + kernelSizeZ; const int num_z_input = last_z - first_z + kernelSizeZ;
for (int p = 0; p < numPlanes; ++p) { for (int p = 0; p < static_cast<int>(numPlanes); ++p) {
const int plane_input_offset = indexMapper.mapGpuInputPlaneToTensorInputOffset(p); const int plane_input_offset = indexMapper.mapGpuInputPlaneToTensorInputOffset(p);
const int plane_kernel_offset = 0; const int plane_kernel_offset = 0;

10
unsupported/Eigen/src/Tensor/TensorExecutor.h
View File

@@ -91,7 +91,7 @@ class TensorExecutor {
TensorEvaluator<Expression, Device> evaluator(expr, device); TensorEvaluator<Expression, Device> evaluator(expr, device);
const bool needs_assign = evaluator.evalSubExprsIfNeeded(NULL); const bool needs_assign = evaluator.evalSubExprsIfNeeded(NULL);
if (needs_assign) { if (needs_assign) {
const StorageIndex size = array_prod(evaluator.dimensions()); const StorageIndex size = static_cast<StorageIndex>(array_prod(evaluator.dimensions()));
for (StorageIndex i = 0; i < size; ++i) { for (StorageIndex i = 0; i < size; ++i) {
evaluator.evalScalar(i); evaluator.evalScalar(i);
} }
@@ -120,7 +120,7 @@ class TensorExecutor<Expression, DefaultDevice, /*Vectorizable=*/true,
TensorEvaluator<Expression, DefaultDevice> evaluator(expr, device); TensorEvaluator<Expression, DefaultDevice> evaluator(expr, device);
const bool needs_assign = evaluator.evalSubExprsIfNeeded(NULL); const bool needs_assign = evaluator.evalSubExprsIfNeeded(NULL);
if (needs_assign) { if (needs_assign) {
const StorageIndex size = array_prod(evaluator.dimensions()); const StorageIndex size = static_cast<StorageIndex>(array_prod(evaluator.dimensions()));
const int PacketSize = const int PacketSize =
unpacket_traits<typename TensorEvaluator<Expression, DefaultDevice>::PacketReturnType>::size; unpacket_traits<typename TensorEvaluator<Expression, DefaultDevice>::PacketReturnType>::size;
@@ -302,7 +302,7 @@ class TensorExecutor<Expression, ThreadPoolDevice, Vectorizable, Tiling> {
Evaluator evaluator(expr, device); Evaluator evaluator(expr, device);
const bool needs_assign = evaluator.evalSubExprsIfNeeded(nullptr); const bool needs_assign = evaluator.evalSubExprsIfNeeded(nullptr);
if (needs_assign) { if (needs_assign) {
const StorageIndex size = array_prod(evaluator.dimensions()); const StorageIndex size = static_cast<StorageIndex>(array_prod(evaluator.dimensions()));
device.parallelFor( device.parallelFor(
size, evaluator.costPerCoeff(Vectorizable), EvalRange::alignBlockSize, size, evaluator.costPerCoeff(Vectorizable), EvalRange::alignBlockSize,
[&evaluator](StorageIndex firstIdx, StorageIndex lastIdx) { EvalRange::run(&evaluator, firstIdx, lastIdx); }); [&evaluator](StorageIndex firstIdx, StorageIndex lastIdx) { EvalRange::run(&evaluator, firstIdx, lastIdx); });
@@ -375,7 +375,7 @@ class TensorAsyncExecutor<Expression, ThreadPoolDevice, DoneCallback, Vectorizab
} }
typedef EvalRange<Evaluator, StorageIndex, Vectorizable> EvalRange; typedef EvalRange<Evaluator, StorageIndex, Vectorizable> EvalRange;
const StorageIndex size = array_prod(ctx->evaluator.dimensions()); const StorageIndex size = static_cast<StorageIndex>(array_prod(ctx->evaluator.dimensions()));
device.parallelForAsync( device.parallelForAsync(
size, ctx->evaluator.costPerCoeff(Vectorizable), EvalRange::alignBlockSize, size, ctx->evaluator.costPerCoeff(Vectorizable), EvalRange::alignBlockSize,
[ctx](StorageIndex firstIdx, StorageIndex lastIdx) { EvalRange::run(&ctx->evaluator, firstIdx, lastIdx); }, [ctx](StorageIndex firstIdx, StorageIndex lastIdx) { EvalRange::run(&ctx->evaluator, firstIdx, lastIdx); },
@@ -583,7 +583,7 @@ EIGEN_STRONG_INLINE void TensorExecutor<Expression, GpuDevice, Vectorizable, Til
numext::mini<int64_t>(device.getNumGpuMultiProcessors() * device.maxGpuThreadsPerMultiProcessor(), numext::mini<int64_t>(device.getNumGpuMultiProcessors() * device.maxGpuThreadsPerMultiProcessor(),
NumTraits<StorageIndex>::highest()) / NumTraits<StorageIndex>::highest()) /
block_size); block_size);
const StorageIndex size = array_prod(evaluator.dimensions()); const StorageIndex size = static_cast<StorageIndex>(array_prod(evaluator.dimensions()));
// Create at least one block to ensure we don't crash with tensors of size 0. // Create at least one block to ensure we don't crash with tensors of size 0.
const int num_blocks = numext::maxi<int>( const int num_blocks = numext::maxi<int>(
numext::mini<int>(max_blocks, static_cast<int>(numext::div_ceil<StorageIndex>(size, block_size))), 1); numext::mini<int>(max_blocks, static_cast<int>(numext::div_ceil<StorageIndex>(size, block_size))), 1);

3
unsupported/Eigen/src/Tensor/TensorMorphing.h
View File

@@ -313,7 +313,8 @@ namespace internal {
// FIXME: Figure out the exact threshold. // FIXME: Figure out the exact threshold.
template <typename Index, typename Device, bool BlockAccess> template <typename Index, typename Device, bool BlockAccess>
struct MemcpyTriggerForSlicing { struct MemcpyTriggerForSlicing {
EIGEN_DEVICE_FUNC MemcpyTriggerForSlicing(const Device& device) : threshold_(2 * device.numThreads()) {} EIGEN_DEVICE_FUNC MemcpyTriggerForSlicing(const Device& device)
: threshold_(static_cast<Index>(2 * device.numThreads())) {}
EIGEN_DEVICE_FUNC bool operator()(Index total, Index contiguous) const { EIGEN_DEVICE_FUNC bool operator()(Index total, Index contiguous) const {
const bool prefer_block_evaluation = BlockAccess && total > 32 * 1024; const bool prefer_block_evaluation = BlockAccess && total > 32 * 1024;
return !prefer_block_evaluation && contiguous > threshold_; return !prefer_block_evaluation && contiguous > threshold_;

6
unsupported/Eigen/src/Tensor/TensorReduction.h
View File

@@ -693,7 +693,7 @@ struct TensorReductionEvaluatorBase<const TensorReductionOp<Op, Dims, ArgType, M
if (internal::InnerReducer<Self, Op, Device>::HasOptimizedImplementation && if (internal::InnerReducer<Self, Op, Device>::HasOptimizedImplementation &&
(reducing_inner_dims || ReducingInnerMostDims)) { (reducing_inner_dims || ReducingInnerMostDims)) {
const Index num_values_to_reduce = internal::array_prod(m_reducedDims); const Index num_values_to_reduce = internal::array_prod(m_reducedDims);
const Index num_coeffs_to_preserve = internal::array_prod(m_dimensions); const Index num_coeffs_to_preserve = static_cast<Index>(internal::array_prod(m_dimensions));
if (!data) { if (!data) {
if ((num_coeffs_to_preserve < 1024 && num_values_to_reduce > num_coeffs_to_preserve && if ((num_coeffs_to_preserve < 1024 && num_values_to_reduce > num_coeffs_to_preserve &&
num_values_to_reduce > 128) || num_values_to_reduce > 128) ||
@@ -729,7 +729,7 @@ struct TensorReductionEvaluatorBase<const TensorReductionOp<Op, Dims, ArgType, M
} }
if (internal::OuterReducer<Self, Op, Device>::HasOptimizedImplementation && preserving_inner_dims) { if (internal::OuterReducer<Self, Op, Device>::HasOptimizedImplementation && preserving_inner_dims) {
const Index num_values_to_reduce = internal::array_prod(m_reducedDims); const Index num_values_to_reduce = internal::array_prod(m_reducedDims);
const Index num_coeffs_to_preserve = internal::array_prod(m_dimensions); const Index num_coeffs_to_preserve = static_cast<Index>(internal::array_prod(m_dimensions));
if (!data) { if (!data) {
if ((num_coeffs_to_preserve < 1024 && num_values_to_reduce > num_coeffs_to_preserve && if ((num_coeffs_to_preserve < 1024 && num_values_to_reduce > num_coeffs_to_preserve &&
num_values_to_reduce > 32) || num_values_to_reduce > 32) ||
@@ -759,7 +759,7 @@ struct TensorReductionEvaluatorBase<const TensorReductionOp<Op, Dims, ArgType, M
// must break into two subexpression and use the SYCL generic Reducer on the device. // must break into two subexpression and use the SYCL generic Reducer on the device.
if (RunningOnSycl) { if (RunningOnSycl) {
const Index num_values_to_reduce = internal::array_prod(m_reducedDims); const Index num_values_to_reduce = internal::array_prod(m_reducedDims);
const Index num_coeffs_to_preserve = internal::array_prod(m_dimensions); const Index num_coeffs_to_preserve = static_cast<Index>(internal::array_prod(m_dimensions));
if (!data) { if (!data) {
data = static_cast<EvaluatorPointerType>( data = static_cast<EvaluatorPointerType>(
m_device.get((CoeffReturnType*)m_device.allocate_temp(sizeof(CoeffReturnType) * num_coeffs_to_preserve))); m_device.get((CoeffReturnType*)m_device.allocate_temp(sizeof(CoeffReturnType) * num_coeffs_to_preserve)));

5
unsupported/Eigen/src/Tensor/TensorReductionGpu.h
View File

@@ -556,6 +556,11 @@ __global__ EIGEN_HIP_LAUNCH_BOUNDS_1024 void InnerReductionKernel(Reducer reduce
} }
} }
#else // EIGEN_CUDA_ARCH >= 300 #else // EIGEN_CUDA_ARCH >= 300
EIGEN_UNUSED_VARIABLE(reducer);
EIGEN_UNUSED_VARIABLE(input);
EIGEN_UNUSED_VARIABLE(num_coeffs_to_reduce);
EIGEN_UNUSED_VARIABLE(num_preserved_coeffs);
EIGEN_UNUSED_VARIABLE(output);
gpu_assert(0 && "Shouldn't be called on unsupported device"); gpu_assert(0 && "Shouldn't be called on unsupported device");
#endif // EIGEN_CUDA_ARCH >= 300 #endif // EIGEN_CUDA_ARCH >= 300
} }

4
unsupported/test/cxx11_tensor_complex_cwise_ops_gpu.cu
View File

@@ -35,8 +35,8 @@ void test_cuda_complex_cwise_ops() {
Eigen::TensorMap<Eigen::Tensor<std::complex<T>, 1, 0, int>, Eigen::Aligned> gpu_in2(d_in2, kNumItems); Eigen::TensorMap<Eigen::Tensor<std::complex<T>, 1, 0, int>, Eigen::Aligned> gpu_in2(d_in2, kNumItems);
Eigen::TensorMap<Eigen::Tensor<std::complex<T>, 1, 0, int>, Eigen::Aligned> gpu_out(d_out, kNumItems); Eigen::TensorMap<Eigen::Tensor<std::complex<T>, 1, 0, int>, Eigen::Aligned> gpu_out(d_out, kNumItems);
const std::complex<T> a(3.14f, 2.7f); const std::complex<T> a(static_cast<T>(3.14), static_cast<T>(2.7));
const std::complex<T> b(-10.6f, 1.4f); const std::complex<T> b(static_cast<T>(-10.6), static_cast<T>(1.4));
gpu_in1.device(gpu_device) = gpu_in1.constant(a); gpu_in1.device(gpu_device) = gpu_in1.constant(a);
gpu_in2.device(gpu_device) = gpu_in2.constant(b); gpu_in2.device(gpu_device) = gpu_in2.constant(b);

4
unsupported/test/cxx11_tensor_device.cu
View File

@@ -176,7 +176,7 @@ void test_3d_convolution(Context* context) {
// Helper method to synchronize device. // Helper method to synchronize device.
template <typename Device> template <typename Device>
void synchronize(Device& device) { /*nothing*/ void synchronize(Device& /*device*/) { /*nothing*/
} }
template <> template <>
void synchronize(Eigen::GpuDevice& device) { void synchronize(Eigen::GpuDevice& device) {
@@ -197,7 +197,7 @@ void test_device_memory(const TensorDevice& device) {
device.memcpyDeviceToHost(host.data(), device_data, count * sizeof(DataType)); device.memcpyDeviceToHost(host.data(), device_data, count * sizeof(DataType));
synchronize(device); synchronize(device);
memset(expected.data(), byte_value, count * sizeof(DataType)); memset(expected.data(), byte_value, count * sizeof(DataType));
for (size_t i = 0; i < count; i++) { for (Index i = 0; i < count; i++) {
VERIFY_IS_EQUAL(host(i), expected(i)); VERIFY_IS_EQUAL(host(i), expected(i));
} }