Reorganize benchmarks into subdirectories and clean up Eigen sources

libeigen/eigen!2176

Co-authored-by: Rasmus Munk Larsen <rmlarsen@gmail.com>

benchmarks/Tuning/CMakeLists.txt

@@ -0,0 +1,12 @@
+eigen_add_benchmark(bench_blocking_sizes bench_blocking_sizes.cpp)
+eigen_add_benchmark(bench_aocl bench_aocl.cpp)
+
+if(BLAS_FOUND)
+  eigen_add_benchmark(bench_blas_gemm bench_blas_gemm.cpp
+    LIBRARIES ${BLAS_LIBRARIES}
+    DEFINITIONS HAVE_BLAS)
+endif()
+
+add_executable(print_blocking print_blocking.cpp)
+target_include_directories(print_blocking PRIVATE ${EIGEN_SOURCE_DIR})
+target_compile_options(print_blocking PRIVATE -O3 -DNDEBUG)

benchmarks/Tuning/bench_aocl.cpp

@@ -0,0 +1,123 @@
+#include <benchmark/benchmark.h>
+#include <Eigen/Core>
+#include <Eigen/Dense>
+#include <Eigen/Eigenvalues>
+
+using namespace Eigen;
+
+static void BM_VectorExp(benchmark::State& state) {
+  int n = state.range(0);
+  VectorXd v = VectorXd::LinSpaced(n, 0.1, 10.0);
+  VectorXd result(n);
+  for (auto _ : state) {
+    result = v.array().exp();
+    benchmark::DoNotOptimize(result.data());
+  }
+  state.SetBytesProcessed(state.iterations() * n * sizeof(double));
+}
+
+static void BM_VectorSin(benchmark::State& state) {
+  int n = state.range(0);
+  VectorXd v = VectorXd::LinSpaced(n, 0.1, 10.0);
+  VectorXd result(n);
+  for (auto _ : state) {
+    result = v.array().sin();
+    benchmark::DoNotOptimize(result.data());
+  }
+  state.SetBytesProcessed(state.iterations() * n * sizeof(double));
+}
+
+static void BM_VectorCos(benchmark::State& state) {
+  int n = state.range(0);
+  VectorXd v = VectorXd::LinSpaced(n, 0.1, 10.0);
+  VectorXd result(n);
+  for (auto _ : state) {
+    result = v.array().cos();
+    benchmark::DoNotOptimize(result.data());
+  }
+  state.SetBytesProcessed(state.iterations() * n * sizeof(double));
+}
+
+static void BM_VectorSqrt(benchmark::State& state) {
+  int n = state.range(0);
+  VectorXd v = VectorXd::LinSpaced(n, 0.1, 10.0);
+  VectorXd result(n);
+  for (auto _ : state) {
+    result = v.array().sqrt();
+    benchmark::DoNotOptimize(result.data());
+  }
+  state.SetBytesProcessed(state.iterations() * n * sizeof(double));
+}
+
+static void BM_VectorLog(benchmark::State& state) {
+  int n = state.range(0);
+  VectorXd v = VectorXd::LinSpaced(n, 0.1, 10.0);
+  VectorXd result(n);
+  for (auto _ : state) {
+    result = v.array().log();
+    benchmark::DoNotOptimize(result.data());
+  }
+  state.SetBytesProcessed(state.iterations() * n * sizeof(double));
+}
+
+static void BM_VectorTanh(benchmark::State& state) {
+  int n = state.range(0);
+  VectorXd v = VectorXd::LinSpaced(n, 0.1, 10.0);
+  VectorXd result(n);
+  for (auto _ : state) {
+    result = v.array().tanh();
+    benchmark::DoNotOptimize(result.data());
+  }
+  state.SetBytesProcessed(state.iterations() * n * sizeof(double));
+}
+
+static void VectorSizes(::benchmark::Benchmark* b) {
+  for (int n : {10000, 100000, 1000000, 5000000}) {
+    b->Arg(n);
+  }
+}
+
+BENCHMARK(BM_VectorExp)->Apply(VectorSizes);
+BENCHMARK(BM_VectorSin)->Apply(VectorSizes);
+BENCHMARK(BM_VectorCos)->Apply(VectorSizes);
+BENCHMARK(BM_VectorSqrt)->Apply(VectorSizes);
+BENCHMARK(BM_VectorLog)->Apply(VectorSizes);
+BENCHMARK(BM_VectorTanh)->Apply(VectorSizes);
+
+static void BM_DGEMM(benchmark::State& state) {
+  int n = state.range(0);
+  MatrixXd A = MatrixXd::Random(n, n);
+  MatrixXd B = MatrixXd::Random(n, n);
+  MatrixXd C(n, n);
+  for (auto _ : state) {
+    C.noalias() = A * B;
+    benchmark::DoNotOptimize(C.data());
+  }
+  state.counters["GFLOPS"] =
+      benchmark::Counter(2.0 * n * n * n, benchmark::Counter::kIsIterationInvariantRate, benchmark::Counter::kIs1000);
+}
+BENCHMARK(BM_DGEMM)->Arg(256)->Arg(512)->Arg(1024)->Arg(2048);
+
+static void BM_EigenDecomposition(benchmark::State& state) {
+  int n = state.range(0);
+  MatrixXd M = MatrixXd::Random(n, n);
+  M = (M + M.transpose()) * 0.5;
+  SelfAdjointEigenSolver<MatrixXd> solver;
+  for (auto _ : state) {
+    solver.compute(M);
+    benchmark::DoNotOptimize(solver.eigenvalues().data());
+  }
+}
+BENCHMARK(BM_EigenDecomposition)->Arg(256)->Arg(512)->Arg(1024);
+
+static void BM_FSI_Risk(benchmark::State& state) {
+  int numPeriods = state.range(0);
+  int numAssets = state.range(1);
+  MatrixXd returns = MatrixXd::Random(numPeriods, numAssets);
+  for (auto _ : state) {
+    MatrixXd cov = (returns.transpose() * returns) / (numPeriods - 1);
+    SelfAdjointEigenSolver<MatrixXd> solver(cov);
+    benchmark::DoNotOptimize(solver.eigenvalues().data());
+  }
+}
+BENCHMARK(BM_FSI_Risk)->Args({10000, 500});

benchmarks/Tuning/bench_blas_gemm.cpp

@@ -0,0 +1,73 @@
+// Benchmark: Eigen GEMM vs CBLAS GEMM
+// Requires CBLAS: compile with -DHAVE_BLAS and link -lcblas
+//
+// Based on bench/benchBlasGemm.cpp
+
+#include <benchmark/benchmark.h>
+#include <Eigen/Core>
+
+using namespace Eigen;
+
+#ifndef SCALAR
+#define SCALAR float
+#endif
+
+typedef SCALAR Scalar;
+typedef Matrix<Scalar, Dynamic, Dynamic> MyMatrix;
+
+static void BM_EigenGemm(benchmark::State& state) {
+  int M = state.range(0);
+  int N = state.range(1);
+  int K = state.range(2);
+  MyMatrix a = MyMatrix::Random(M, K);
+  MyMatrix b = MyMatrix::Random(K, N);
+  MyMatrix c = MyMatrix::Random(M, N);
+  for (auto _ : state) {
+    c.noalias() += a * b;
+    benchmark::DoNotOptimize(c.data());
+  }
+  state.counters["GFLOPS"] =
+      benchmark::Counter(2.0 * M * N * K, benchmark::Counter::kIsIterationInvariantRate, benchmark::Counter::kIs1000);
+}
+
+#ifdef HAVE_BLAS
+extern "C" {
+#include <cblas.h>
+}
+
+#ifdef _FLOAT
+#define CBLAS_GEMM cblas_sgemm
+#else
+#define CBLAS_GEMM cblas_dgemm
+#endif
+
+static void BM_CblasGemm(benchmark::State& state) {
+  int M = state.range(0);
+  int N = state.range(1);
+  int K = state.range(2);
+  MyMatrix a = MyMatrix::Random(M, K);
+  MyMatrix b = MyMatrix::Random(K, N);
+  MyMatrix c = MyMatrix::Random(M, N);
+  Scalar alpha = 1, beta = 1;
+  for (auto _ : state) {
+    CBLAS_GEMM(CblasColMajor, CblasNoTrans, CblasNoTrans, M, N, K, alpha, a.data(), M, b.data(), K, beta, c.data(), M);
+    benchmark::DoNotOptimize(c.data());
+  }
+  state.counters["GFLOPS"] =
+      benchmark::Counter(2.0 * M * N * K, benchmark::Counter::kIsIterationInvariantRate, benchmark::Counter::kIs1000);
+}
+#endif
+
+static void GemmSizes(::benchmark::Benchmark* b) {
+  for (int s : {32, 64, 128, 256, 512, 1024, 2048}) {
+    b->Args({s, s, s});
+  }
+  // Rectangular
+  b->Args({1000, 100, 1000});
+  b->Args({100, 1000, 100});
+}
+
+BENCHMARK(BM_EigenGemm)->Apply(GemmSizes);
+#ifdef HAVE_BLAS
+BENCHMARK(BM_CblasGemm)->Apply(GemmSizes);
+#endif

benchmarks/Tuning/bench_blocking_sizes.cpp

@@ -0,0 +1,76 @@
+#include <benchmark/benchmark.h>
+
+#include <cstdint>
+
+bool eigen_use_specific_block_size;
+int eigen_block_size_k, eigen_block_size_m, eigen_block_size_n;
+#define EIGEN_TEST_SPECIFIC_BLOCKING_SIZES eigen_use_specific_block_size
+#define EIGEN_TEST_SPECIFIC_BLOCKING_SIZE_K eigen_block_size_k
+#define EIGEN_TEST_SPECIFIC_BLOCKING_SIZE_M eigen_block_size_m
+#define EIGEN_TEST_SPECIFIC_BLOCKING_SIZE_N eigen_block_size_n
+#include <Eigen/Core>
+
+using namespace Eigen;
+
+typedef MatrixXf MatrixType;
+typedef MatrixType::Scalar Scalar;
+
+static void BM_GemmDefaultBlocking(benchmark::State& state) {
+  int k = state.range(0);
+  int m = state.range(1);
+  int n = state.range(2);
+  eigen_use_specific_block_size = false;
+  MatrixType lhs = MatrixType::Random(m, k);
+  MatrixType rhs = MatrixType::Random(k, n);
+  MatrixType dst = MatrixType::Zero(m, n);
+  for (auto _ : state) {
+    dst.noalias() = lhs * rhs;
+    benchmark::DoNotOptimize(dst.data());
+  }
+  state.counters["GFLOPS"] =
+      benchmark::Counter(2.0 * k * m * n, benchmark::Counter::kIsIterationInvariantRate, benchmark::Counter::kIs1000);
+}
+
+static void BM_GemmCustomBlocking(benchmark::State& state) {
+  int k = state.range(0);
+  int m = state.range(1);
+  int n = state.range(2);
+  int bk = state.range(3);
+  int bm = state.range(4);
+  int bn = state.range(5);
+  eigen_use_specific_block_size = true;
+  eigen_block_size_k = bk;
+  eigen_block_size_m = bm;
+  eigen_block_size_n = bn;
+  MatrixType lhs = MatrixType::Random(m, k);
+  MatrixType rhs = MatrixType::Random(k, n);
+  MatrixType dst = MatrixType::Zero(m, n);
+  for (auto _ : state) {
+    dst.noalias() = lhs * rhs;
+    benchmark::DoNotOptimize(dst.data());
+  }
+  state.counters["GFLOPS"] =
+      benchmark::Counter(2.0 * k * m * n, benchmark::Counter::kIsIterationInvariantRate, benchmark::Counter::kIs1000);
+}
+
+static void DefaultBlockingSizes(::benchmark::Benchmark* b) {
+  for (int s : {64, 128, 256, 512, 1024, 2048}) {
+    b->Args({s, s, s});
+  }
+}
+
+static void CustomBlockingSizes(::benchmark::Benchmark* b) {
+  // Test a few product sizes with varying block sizes
+  for (int s : {256, 512, 1024}) {
+    for (int bk : {16, 32, 64, 128, 256}) {
+      if (bk > s) continue;
+      for (int bm : {16, 32, 64, 128, 256}) {
+        if (bm > s) continue;
+        b->Args({s, s, s, bk, bm, s});
+      }
+    }
+  }
+}
+
+BENCHMARK(BM_GemmDefaultBlocking)->Apply(DefaultBlockingSizes);
+BENCHMARK(BM_GemmCustomBlocking)->Apply(CustomBlockingSizes);

benchmarks/Tuning/print_blocking.cpp

@@ -0,0 +1,51 @@
+#include <Eigen/Core>
+#include <cstdio>
+
+using namespace Eigen;
+using namespace Eigen::internal;
+
+int main() {
+  printf("%-8s %-8s %-8s | %-8s %-8s %-8s | %-8s %-8s %-8s | %-8s %-8s %-8s\n", "m", "n", "k", "kc_f", "mc_f", "nc_f",
+         "kc_d", "mc_d", "nc_d", "mr_f", "nr_f", "LhsPr_f");
+
+  // Print gebp_traits info
+  {
+    using Traits = gebp_traits<float, float>;
+    printf("Float traits: mr=%d, nr=%d, LhsProgress=%d, RhsProgress=%d, NumberOfRegisters=%d\n", Traits::mr, Traits::nr,
+           Traits::LhsProgress, Traits::RhsProgress, Traits::NumberOfRegisters);
+  }
+  {
+    using Traits = gebp_traits<double, double>;
+    printf("Double traits: mr=%d, nr=%d, LhsProgress=%d, RhsProgress=%d, NumberOfRegisters=%d\n", Traits::mr,
+           Traits::nr, Traits::LhsProgress, Traits::RhsProgress, Traits::NumberOfRegisters);
+  }
+
+  // Print cache sizes
+  std::ptrdiff_t l1, l2, l3;
+  manage_caching_sizes(GetAction, &l1, &l2, &l3);
+  printf("Cache sizes: L1=%ld, L2=%ld, L3=%ld\n", (long)l1, (long)l2, (long)l3);
+
+  for (int size : {8, 16, 32, 64, 96, 128, 160, 192, 224, 256, 288, 320, 384, 448, 512, 768, 1024, 1536, 2048}) {
+    {
+      Index kf = size, mf = size, nf = size;
+      computeProductBlockingSizes<float, float>(kf, mf, nf);
+      Index kd = size, md = size, nd = size;
+      computeProductBlockingSizes<double, double>(kd, md, nd);
+      printf("%-8d %-8d %-8d | %-8ld %-8ld %-8ld | %-8ld %-8ld %-8ld\n", size, size, size, (long)kf, (long)mf, (long)nf,
+             (long)kd, (long)md, (long)nd);
+    }
+  }
+
+  // Non-square
+  for (auto [m, n, k] :
+       std::initializer_list<std::tuple<int, int, int>>{{64, 64, 1024}, {1024, 64, 64}, {64, 1024, 64}}) {
+    Index kf = k, mf = m, nf = n;
+    computeProductBlockingSizes<float, float>(kf, mf, nf);
+    Index kd = k, md = m, nd = n;
+    computeProductBlockingSizes<double, double>(kd, md, nd);
+    printf("%-8d %-8d %-8d | %-8ld %-8ld %-8ld | %-8ld %-8ld %-8ld\n", m, n, k, (long)kf, (long)mf, (long)nf, (long)kd,
+           (long)md, (long)nd);
+  }
+
+  return 0;
+}