mirror of
https://gitlab.com/libeigen/eigen.git
synced 2026-04-10 11:34:33 +08:00
GPU: Add library dispatch module (DeviceMatrix, cuBLAS, cuSOLVER)
Add Eigen/GPU module: A standalone GPU library dispatch layer where DeviceMatrix<Scalar> operations map 1:1 to cuBLAS/cuSOLVER calls. CPU and GPU solvers coexist in the same binary with compatible syntax. Core infrastructure: - DeviceMatrix<Scalar>: RAII dense column-major GPU memory wrapper with async host transfer (fromHost/toHost) and CUDA event-based cross-stream synchronization. - GpuContext: Unified execution context owning a CUDA stream + cuBLAS handle + cuSOLVER handle. Thread-local default with explicit override via setThreadLocal(). Stream-borrowing constructor for integration. - DeviceBuffer: Typed RAII device allocation with move semantics. cuBLAS dispatch (expression syntax): - GEMM: d_C = d_A.adjoint() * d_B (cublasXgemm) - TRSM: d_X = d_A.triangularView<Lower>().solve(d_B) (cublasXtrsm) - SYMM/HEMM: d_C = d_A.selfadjointView<Lower>() * d_B (cublasXsymm) - SYRK/HERK: d_C = d_A * d_A.adjoint() (cublasXsyrk) cuSOLVER dispatch: - GpuLLT: Cached Cholesky factorization (cusolverDnXpotrf + Xpotrs) - GpuLU: Cached LU factorization (cusolverDnXgetrf + Xgetrs) - Solver chaining: auto x = d_A.llt().solve(d_B) - Solver expressions with .device(ctx) for explicit stream control. CI: Bump CUDA container to Ubuntu 22.04 (CMake 3.22), GCC 10->11, Clang 12->14. Bump cmake_minimum_required to 3.17 for FindCUDAToolkit. Tests: gpu_cublas.cpp, gpu_cusolver_llt.cpp, gpu_cusolver_lu.cpp, gpu_device_matrix.cpp, gpu_library_example.cu Benchmarks: bench_gpu_solvers.cpp, bench_gpu_chaining.cpp, bench_gpu_batching.cpp
This commit is contained in:
Rasmus Munk Larsen
@@ -479,6 +479,78 @@ if(CUDA_FOUND AND EIGEN_TEST_CUDA)
|
||||
|
||||
ei_add_test(gpu_example)
|
||||
ei_add_test(gpu_basic)
|
||||
ei_add_test(gpu_library_example "" "CUDA::cusolver")
|
||||
|
||||
# DeviceMatrix tests: only CUDA runtime, no NVIDIA libraries.
|
||||
unset(EIGEN_ADD_TEST_FILENAME_EXTENSION)
|
||||
add_executable(gpu_device_matrix gpu_device_matrix.cpp)
|
||||
target_include_directories(gpu_device_matrix PRIVATE
|
||||
"${CUDA_TOOLKIT_ROOT_DIR}/include"
|
||||
"${CMAKE_CURRENT_BINARY_DIR}")
|
||||
target_link_libraries(gpu_device_matrix Eigen3::Eigen CUDA::cudart)
|
||||
target_compile_definitions(gpu_device_matrix PRIVATE
|
||||
EIGEN_TEST_MAX_SIZE=${EIGEN_TEST_MAX_SIZE}
|
||||
EIGEN_TEST_PART_ALL=1)
|
||||
add_test(NAME gpu_device_matrix COMMAND gpu_device_matrix)
|
||||
add_dependencies(buildtests gpu_device_matrix)
|
||||
add_dependencies(buildtests_gpu gpu_device_matrix)
|
||||
set_property(TEST gpu_device_matrix APPEND PROPERTY LABELS "Official;gpu")
|
||||
set_property(TEST gpu_device_matrix PROPERTY SKIP_RETURN_CODE 77)
|
||||
set(EIGEN_ADD_TEST_FILENAME_EXTENSION "cu")
|
||||
|
||||
# Library-specific GPU tests (activated by later phases, OFF by default).
|
||||
# CUDAToolkit imported targets (CUDA::cublas, etc.) are available from
|
||||
# find_package(CUDAToolkit) above.
|
||||
option(EIGEN_TEST_CUBLAS "Test cuBLAS integration" OFF)
|
||||
if(EIGEN_TEST_CUBLAS AND TARGET CUDA::cublas)
|
||||
# cuBLAS tests are plain .cpp files (no device code), like cuSOLVER tests.
|
||||
unset(EIGEN_ADD_TEST_FILENAME_EXTENSION)
|
||||
add_executable(gpu_cublas gpu_cublas.cpp)
|
||||
target_include_directories(gpu_cublas PRIVATE
|
||||
"${CUDA_TOOLKIT_ROOT_DIR}/include"
|
||||
"${CMAKE_CURRENT_BINARY_DIR}")
|
||||
target_link_libraries(gpu_cublas
|
||||
Eigen3::Eigen CUDA::cudart CUDA::cublas CUDA::cusolver)
|
||||
target_compile_definitions(gpu_cublas PRIVATE
|
||||
EIGEN_TEST_MAX_SIZE=${EIGEN_TEST_MAX_SIZE}
|
||||
EIGEN_TEST_PART_ALL=1)
|
||||
add_test(NAME gpu_cublas COMMAND gpu_cublas)
|
||||
add_dependencies(buildtests gpu_cublas)
|
||||
add_dependencies(buildtests_gpu gpu_cublas)
|
||||
set_property(TEST gpu_cublas APPEND PROPERTY LABELS "Official;gpu")
|
||||
set_property(TEST gpu_cublas PROPERTY SKIP_RETURN_CODE 77)
|
||||
set(EIGEN_ADD_TEST_FILENAME_EXTENSION "cu")
|
||||
endif()
|
||||
|
||||
option(EIGEN_TEST_CUSOLVER "Test cuSOLVER integration" OFF)
|
||||
if(EIGEN_TEST_CUSOLVER AND TARGET CUDA::cusolver)
|
||||
# cuSOLVER tests are plain .cpp files: no device code, compiled by the host
|
||||
# compiler and linked against CUDA runtime + cuSOLVER. This avoids NVCC
|
||||
# instantiating Eigen's CPU packet operations for CUDA vector types.
|
||||
unset(EIGEN_ADD_TEST_FILENAME_EXTENSION)
|
||||
foreach(_cusolver_test IN ITEMS gpu_cusolver_llt gpu_cusolver_lu)
|
||||
add_executable(${_cusolver_test} ${_cusolver_test}.cpp)
|
||||
target_include_directories(${_cusolver_test} PRIVATE
|
||||
"${CUDA_TOOLKIT_ROOT_DIR}/include"
|
||||
"${CMAKE_CURRENT_BINARY_DIR}")
|
||||
target_link_libraries(${_cusolver_test}
|
||||
Eigen3::Eigen CUDA::cudart CUDA::cusolver CUDA::cublas)
|
||||
target_compile_definitions(${_cusolver_test} PRIVATE
|
||||
EIGEN_TEST_MAX_SIZE=${EIGEN_TEST_MAX_SIZE}
|
||||
EIGEN_TEST_PART_ALL=1)
|
||||
add_test(NAME ${_cusolver_test} COMMAND "${_cusolver_test}")
|
||||
add_dependencies(buildtests ${_cusolver_test})
|
||||
add_dependencies(buildtests_gpu ${_cusolver_test})
|
||||
set_property(TEST ${_cusolver_test} APPEND PROPERTY LABELS "Official;gpu")
|
||||
set_property(TEST ${_cusolver_test} PROPERTY SKIP_RETURN_CODE 77)
|
||||
endforeach()
|
||||
set(EIGEN_ADD_TEST_FILENAME_EXTENSION "cu")
|
||||
endif()
|
||||
|
||||
option(EIGEN_TEST_CUSPARSE "Test cuSPARSE integration" OFF)
|
||||
if(EIGEN_TEST_CUSPARSE AND TARGET CUDA::cusparse)
|
||||
ei_add_test(gpu_cusparse "" "CUDA::cusparse")
|
||||
endif()
|
||||
|
||||
unset(EIGEN_ADD_TEST_FILENAME_EXTENSION)
|
||||
|
||||
|
||||
72
test/gpu_context.h
Normal file
72
test/gpu_context.h
Normal file
@@ -0,0 +1,72 @@
|
||||
// This file is part of Eigen, a lightweight C++ template library
|
||||
// for linear algebra.
|
||||
//
|
||||
// Copyright (C) 2026 Rasmus Munk Larsen <rmlarsen@gmail.com>
|
||||
//
|
||||
// This Source Code Form is subject to the terms of the Mozilla
|
||||
// Public License v. 2.0. If a copy of the MPL was not distributed
|
||||
// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
|
||||
#ifndef EIGEN_TEST_GPU_CONTEXT_H
|
||||
#define EIGEN_TEST_GPU_CONTEXT_H
|
||||
|
||||
// RAII context for GPU tests that use NVIDIA library APIs (cuBLAS, cuSOLVER, etc.).
|
||||
// Owns a non-default CUDA stream. Library handles (cuBLAS, cuSOLVER, etc.) are added
|
||||
// here by each integration phase as needed; each handle is bound to the owned stream.
|
||||
//
|
||||
// Usage:
|
||||
// GpuContext ctx;
|
||||
// auto buf = gpu_copy_to_device(ctx.stream, A);
|
||||
// // ... call NVIDIA library APIs using ctx.stream / ctx.cusolver ...
|
||||
// ctx.synchronize();
|
||||
|
||||
#include "gpu_test_helper.h"
|
||||
|
||||
#ifdef EIGEN_USE_GPU
|
||||
#include <cusolverDn.h>
|
||||
|
||||
// Checks cuSOLVER return codes, aborts on failure.
|
||||
#define CUSOLVER_CHECK(expr) \
|
||||
do { \
|
||||
cusolverStatus_t _status = (expr); \
|
||||
if (_status != CUSOLVER_STATUS_SUCCESS) { \
|
||||
printf("cuSOLVER error %d at %s:%d\n", static_cast<int>(_status), __FILE__, __LINE__); \
|
||||
gpu_assert(false); \
|
||||
} \
|
||||
} while (0)
|
||||
|
||||
struct GpuContext {
|
||||
cudaStream_t stream = nullptr;
|
||||
cusolverDnHandle_t cusolver = nullptr;
|
||||
|
||||
GpuContext() {
|
||||
GPU_CHECK(gpuGetDevice(&device_));
|
||||
GPU_CHECK(gpuGetDeviceProperties(&device_props_, device_));
|
||||
GPU_CHECK(cudaStreamCreate(&stream));
|
||||
CUSOLVER_CHECK(cusolverDnCreate(&cusolver));
|
||||
CUSOLVER_CHECK(cusolverDnSetStream(cusolver, stream));
|
||||
}
|
||||
|
||||
~GpuContext() {
|
||||
if (cusolver) CUSOLVER_CHECK(cusolverDnDestroy(cusolver));
|
||||
if (stream) GPU_CHECK(cudaStreamDestroy(stream));
|
||||
}
|
||||
|
||||
int device() const { return device_; }
|
||||
const gpuDeviceProp_t& deviceProperties() const { return device_props_; }
|
||||
|
||||
// Wait for all work submitted on this context's stream to complete.
|
||||
void synchronize() { GPU_CHECK(cudaStreamSynchronize(stream)); }
|
||||
|
||||
// Non-copyable, non-movable.
|
||||
GpuContext(const GpuContext&) = delete;
|
||||
GpuContext& operator=(const GpuContext&) = delete;
|
||||
|
||||
private:
|
||||
int device_ = 0;
|
||||
gpuDeviceProp_t device_props_;
|
||||
};
|
||||
|
||||
#endif // EIGEN_USE_GPU
|
||||
|
||||
#endif // EIGEN_TEST_GPU_CONTEXT_H
|
||||
728
test/gpu_cublas.cpp
Normal file
728
test/gpu_cublas.cpp
Normal file
@@ -0,0 +1,728 @@
|
||||
// This file is part of Eigen, a lightweight C++ template library
|
||||
// for linear algebra.
|
||||
//
|
||||
// Copyright (C) 2026 Rasmus Munk Larsen <rmlarsen@gmail.com>
|
||||
//
|
||||
// This Source Code Form is subject to the terms of the Mozilla
|
||||
// Public License v. 2.0. If a copy of the MPL was not distributed
|
||||
// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
|
||||
// Tests for cuBLAS GEMM dispatch via DeviceMatrix expression syntax.
|
||||
// Covers: d_C = d_A * d_B, adjoint, transpose, scaled, +=, .device(ctx).
|
||||
|
||||
#define EIGEN_USE_GPU
|
||||
#include "main.h"
|
||||
#include <Eigen/GPU>
|
||||
|
||||
using namespace Eigen;
|
||||
|
||||
// ---- Basic GEMM: C = A * B -------------------------------------------------
|
||||
|
||||
template <typename Scalar>
|
||||
void test_gemm_basic(Index m, Index n, Index k) {
|
||||
using Mat = Matrix<Scalar, Dynamic, Dynamic>;
|
||||
using RealScalar = typename NumTraits<Scalar>::Real;
|
||||
|
||||
Mat A = Mat::Random(m, k);
|
||||
Mat B = Mat::Random(k, n);
|
||||
|
||||
auto d_A = DeviceMatrix<Scalar>::fromHost(A);
|
||||
auto d_B = DeviceMatrix<Scalar>::fromHost(B);
|
||||
|
||||
// Expression: d_C = d_A * d_B
|
||||
DeviceMatrix<Scalar> d_C;
|
||||
d_C = d_A * d_B;
|
||||
|
||||
Mat C = d_C.toHost();
|
||||
Mat C_ref = A * B;
|
||||
|
||||
RealScalar tol = RealScalar(k) * NumTraits<Scalar>::epsilon() * C_ref.norm();
|
||||
VERIFY((C - C_ref).norm() < tol);
|
||||
}
|
||||
|
||||
// ---- GEMM with adjoint: C = A^H * B ----------------------------------------
|
||||
|
||||
template <typename Scalar>
|
||||
void test_gemm_adjoint_lhs(Index m, Index n, Index k) {
|
||||
using Mat = Matrix<Scalar, Dynamic, Dynamic>;
|
||||
using RealScalar = typename NumTraits<Scalar>::Real;
|
||||
|
||||
Mat A = Mat::Random(k, m); // A is k×m, A^H is m×k
|
||||
Mat B = Mat::Random(k, n);
|
||||
|
||||
auto d_A = DeviceMatrix<Scalar>::fromHost(A);
|
||||
auto d_B = DeviceMatrix<Scalar>::fromHost(B);
|
||||
|
||||
DeviceMatrix<Scalar> d_C;
|
||||
d_C = d_A.adjoint() * d_B;
|
||||
|
||||
Mat C = d_C.toHost();
|
||||
Mat C_ref = A.adjoint() * B;
|
||||
|
||||
RealScalar tol = RealScalar(k) * NumTraits<Scalar>::epsilon() * C_ref.norm();
|
||||
VERIFY((C - C_ref).norm() < tol);
|
||||
}
|
||||
|
||||
// ---- GEMM with transpose: C = A * B^T --------------------------------------
|
||||
|
||||
template <typename Scalar>
|
||||
void test_gemm_transpose_rhs(Index m, Index n, Index k) {
|
||||
using Mat = Matrix<Scalar, Dynamic, Dynamic>;
|
||||
using RealScalar = typename NumTraits<Scalar>::Real;
|
||||
|
||||
Mat A = Mat::Random(m, k);
|
||||
Mat B = Mat::Random(n, k); // B is n×k, B^T is k×n
|
||||
|
||||
auto d_A = DeviceMatrix<Scalar>::fromHost(A);
|
||||
auto d_B = DeviceMatrix<Scalar>::fromHost(B);
|
||||
|
||||
DeviceMatrix<Scalar> d_C;
|
||||
d_C = d_A * d_B.transpose();
|
||||
|
||||
Mat C = d_C.toHost();
|
||||
Mat C_ref = A * B.transpose();
|
||||
|
||||
RealScalar tol = RealScalar(k) * NumTraits<Scalar>::epsilon() * C_ref.norm();
|
||||
VERIFY((C - C_ref).norm() < tol);
|
||||
}
|
||||
|
||||
// ---- GEMM with scaled: C = alpha * A * B ------------------------------------
|
||||
|
||||
template <typename Scalar>
|
||||
void test_gemm_scaled(Index m, Index n, Index k) {
|
||||
using Mat = Matrix<Scalar, Dynamic, Dynamic>;
|
||||
using RealScalar = typename NumTraits<Scalar>::Real;
|
||||
|
||||
Mat A = Mat::Random(m, k);
|
||||
Mat B = Mat::Random(k, n);
|
||||
Scalar alpha = Scalar(2.5);
|
||||
|
||||
auto d_A = DeviceMatrix<Scalar>::fromHost(A);
|
||||
auto d_B = DeviceMatrix<Scalar>::fromHost(B);
|
||||
|
||||
DeviceMatrix<Scalar> d_C;
|
||||
d_C = alpha * d_A * d_B;
|
||||
|
||||
Mat C = d_C.toHost();
|
||||
Mat C_ref = alpha * A * B;
|
||||
|
||||
RealScalar tol = RealScalar(k) * NumTraits<Scalar>::epsilon() * C_ref.norm();
|
||||
VERIFY((C - C_ref).norm() < tol);
|
||||
}
|
||||
|
||||
// ---- GEMM accumulate: C += A * B (beta=1) -----------------------------------
|
||||
|
||||
template <typename Scalar>
|
||||
void test_gemm_accumulate(Index m, Index n, Index k) {
|
||||
using Mat = Matrix<Scalar, Dynamic, Dynamic>;
|
||||
using RealScalar = typename NumTraits<Scalar>::Real;
|
||||
|
||||
Mat A = Mat::Random(m, k);
|
||||
Mat B = Mat::Random(k, n);
|
||||
Mat C_init = Mat::Random(m, n);
|
||||
|
||||
auto d_A = DeviceMatrix<Scalar>::fromHost(A);
|
||||
auto d_B = DeviceMatrix<Scalar>::fromHost(B);
|
||||
auto d_C = DeviceMatrix<Scalar>::fromHost(C_init);
|
||||
|
||||
d_C += d_A * d_B;
|
||||
|
||||
Mat C = d_C.toHost();
|
||||
Mat C_ref = C_init + A * B;
|
||||
|
||||
RealScalar tol = RealScalar(k) * NumTraits<Scalar>::epsilon() * C_ref.norm();
|
||||
VERIFY((C - C_ref).norm() < tol);
|
||||
}
|
||||
|
||||
// ---- GEMM accumulate into empty destination ---------------------------------
|
||||
|
||||
template <typename Scalar>
|
||||
void test_gemm_accumulate_empty(Index m, Index n, Index k) {
|
||||
using Mat = Matrix<Scalar, Dynamic, Dynamic>;
|
||||
using RealScalar = typename NumTraits<Scalar>::Real;
|
||||
|
||||
Mat A = Mat::Random(m, k);
|
||||
Mat B = Mat::Random(k, n);
|
||||
|
||||
auto d_A = DeviceMatrix<Scalar>::fromHost(A);
|
||||
auto d_B = DeviceMatrix<Scalar>::fromHost(B);
|
||||
DeviceMatrix<Scalar> d_C;
|
||||
|
||||
d_C += d_A * d_B;
|
||||
|
||||
Mat C = d_C.toHost();
|
||||
Mat C_ref = A * B;
|
||||
|
||||
RealScalar tol = RealScalar(k) * NumTraits<Scalar>::epsilon() * C_ref.norm();
|
||||
VERIFY((C - C_ref).norm() < tol);
|
||||
}
|
||||
|
||||
// ---- GEMM subtract: C -= A * B (beta=1, alpha=-1) --------------------------
|
||||
|
||||
template <typename Scalar>
|
||||
void test_gemm_subtract(Index m, Index n, Index k) {
|
||||
using Mat = Matrix<Scalar, Dynamic, Dynamic>;
|
||||
using RealScalar = typename NumTraits<Scalar>::Real;
|
||||
|
||||
Mat A = Mat::Random(m, k);
|
||||
Mat B = Mat::Random(k, n);
|
||||
Mat C_init = Mat::Random(m, n);
|
||||
|
||||
auto d_A = DeviceMatrix<Scalar>::fromHost(A);
|
||||
auto d_B = DeviceMatrix<Scalar>::fromHost(B);
|
||||
auto d_C = DeviceMatrix<Scalar>::fromHost(C_init);
|
||||
|
||||
GpuContext ctx;
|
||||
d_C.device(ctx) -= d_A * d_B;
|
||||
|
||||
Mat C = d_C.toHost();
|
||||
Mat C_ref = C_init - A * B;
|
||||
|
||||
RealScalar tol = RealScalar(k) * NumTraits<Scalar>::epsilon() * C_ref.norm();
|
||||
VERIFY((C - C_ref).norm() < tol);
|
||||
}
|
||||
|
||||
// ---- GEMM subtract from empty destination -----------------------------------
|
||||
|
||||
template <typename Scalar>
|
||||
void test_gemm_subtract_empty(Index m, Index n, Index k) {
|
||||
using Mat = Matrix<Scalar, Dynamic, Dynamic>;
|
||||
using RealScalar = typename NumTraits<Scalar>::Real;
|
||||
|
||||
Mat A = Mat::Random(m, k);
|
||||
Mat B = Mat::Random(k, n);
|
||||
|
||||
auto d_A = DeviceMatrix<Scalar>::fromHost(A);
|
||||
auto d_B = DeviceMatrix<Scalar>::fromHost(B);
|
||||
|
||||
GpuContext ctx;
|
||||
DeviceMatrix<Scalar> d_C;
|
||||
d_C.device(ctx) -= d_A * d_B;
|
||||
|
||||
Mat C = d_C.toHost();
|
||||
Mat C_ref = -(A * B);
|
||||
|
||||
RealScalar tol = RealScalar(k) * NumTraits<Scalar>::epsilon() * C_ref.norm();
|
||||
VERIFY((C - C_ref).norm() < tol);
|
||||
}
|
||||
|
||||
// ---- GEMM with scaled RHS: C = A * (alpha * B) -----------------------------
|
||||
|
||||
template <typename Scalar>
|
||||
void test_gemm_scaled_rhs(Index m, Index n, Index k) {
|
||||
using Mat = Matrix<Scalar, Dynamic, Dynamic>;
|
||||
using RealScalar = typename NumTraits<Scalar>::Real;
|
||||
|
||||
Mat A = Mat::Random(m, k);
|
||||
Mat B = Mat::Random(k, n);
|
||||
Scalar alpha = Scalar(3.0);
|
||||
|
||||
auto d_A = DeviceMatrix<Scalar>::fromHost(A);
|
||||
auto d_B = DeviceMatrix<Scalar>::fromHost(B);
|
||||
|
||||
DeviceMatrix<Scalar> d_C;
|
||||
d_C = d_A * (alpha * d_B);
|
||||
|
||||
Mat C = d_C.toHost();
|
||||
Mat C_ref = A * (alpha * B);
|
||||
|
||||
RealScalar tol = RealScalar(k) * NumTraits<Scalar>::epsilon() * C_ref.norm();
|
||||
VERIFY((C - C_ref).norm() < tol);
|
||||
}
|
||||
|
||||
// ---- GEMM dimension mismatch must assert ------------------------------------
|
||||
|
||||
template <typename Scalar>
|
||||
void test_gemm_dimension_mismatch() {
|
||||
using Mat = Matrix<Scalar, Dynamic, Dynamic>;
|
||||
|
||||
Mat A = Mat::Random(8, 5);
|
||||
Mat B = Mat::Random(6, 7); // inner dimension mismatch
|
||||
|
||||
auto d_A = DeviceMatrix<Scalar>::fromHost(A);
|
||||
auto d_B = DeviceMatrix<Scalar>::fromHost(B);
|
||||
DeviceMatrix<Scalar> d_C;
|
||||
|
||||
VERIFY_RAISES_ASSERT(d_C = d_A * d_B);
|
||||
}
|
||||
|
||||
// ---- GEMM with explicit GpuContext ------------------------------------------
|
||||
|
||||
template <typename Scalar>
|
||||
void test_gemm_explicit_context(Index m, Index n, Index k) {
|
||||
using Mat = Matrix<Scalar, Dynamic, Dynamic>;
|
||||
using RealScalar = typename NumTraits<Scalar>::Real;
|
||||
|
||||
Mat A = Mat::Random(m, k);
|
||||
Mat B = Mat::Random(k, n);
|
||||
|
||||
auto d_A = DeviceMatrix<Scalar>::fromHost(A);
|
||||
auto d_B = DeviceMatrix<Scalar>::fromHost(B);
|
||||
|
||||
GpuContext ctx;
|
||||
DeviceMatrix<Scalar> d_C;
|
||||
d_C.device(ctx) = d_A * d_B;
|
||||
|
||||
Mat C = d_C.toHost();
|
||||
Mat C_ref = A * B;
|
||||
|
||||
RealScalar tol = RealScalar(k) * NumTraits<Scalar>::epsilon() * C_ref.norm();
|
||||
VERIFY((C - C_ref).norm() < tol);
|
||||
}
|
||||
|
||||
// ---- GEMM cross-context reuse of the same destination -----------------------
|
||||
|
||||
template <typename Scalar>
|
||||
void test_gemm_cross_context_reuse(Index n) {
|
||||
using Mat = Matrix<Scalar, Dynamic, Dynamic>;
|
||||
using RealScalar = typename NumTraits<Scalar>::Real;
|
||||
|
||||
Mat A = Mat::Random(n, n);
|
||||
Mat B = Mat::Random(n, n);
|
||||
Mat D = Mat::Random(n, n);
|
||||
Mat E = Mat::Random(n, n);
|
||||
|
||||
auto d_A = DeviceMatrix<Scalar>::fromHost(A);
|
||||
auto d_B = DeviceMatrix<Scalar>::fromHost(B);
|
||||
auto d_D = DeviceMatrix<Scalar>::fromHost(D);
|
||||
auto d_E = DeviceMatrix<Scalar>::fromHost(E);
|
||||
|
||||
GpuContext ctx1;
|
||||
GpuContext ctx2;
|
||||
DeviceMatrix<Scalar> d_C;
|
||||
d_C.device(ctx1) = d_A * d_B;
|
||||
d_C.device(ctx2) += d_D * d_E;
|
||||
|
||||
Mat C = d_C.toHost();
|
||||
Mat C_ref = A * B + D * E;
|
||||
|
||||
RealScalar tol = RealScalar(2) * RealScalar(n) * NumTraits<Scalar>::epsilon() * C_ref.norm();
|
||||
VERIFY((C - C_ref).norm() < tol);
|
||||
}
|
||||
|
||||
// ---- GEMM cross-context resize of the destination ---------------------------
|
||||
|
||||
template <typename Scalar>
|
||||
void test_gemm_cross_context_resize() {
|
||||
using Mat = Matrix<Scalar, Dynamic, Dynamic>;
|
||||
using RealScalar = typename NumTraits<Scalar>::Real;
|
||||
|
||||
Mat A = Mat::Random(64, 64);
|
||||
Mat B = Mat::Random(64, 64);
|
||||
Mat D = Mat::Random(32, 16);
|
||||
Mat E = Mat::Random(16, 8);
|
||||
|
||||
auto d_A = DeviceMatrix<Scalar>::fromHost(A);
|
||||
auto d_B = DeviceMatrix<Scalar>::fromHost(B);
|
||||
auto d_D = DeviceMatrix<Scalar>::fromHost(D);
|
||||
auto d_E = DeviceMatrix<Scalar>::fromHost(E);
|
||||
|
||||
GpuContext ctx1;
|
||||
GpuContext ctx2;
|
||||
DeviceMatrix<Scalar> d_C;
|
||||
d_C.device(ctx1) = d_A * d_B;
|
||||
d_C.device(ctx2) = d_D * d_E;
|
||||
|
||||
Mat C = d_C.toHost();
|
||||
Mat C_ref = D * E;
|
||||
|
||||
RealScalar tol = RealScalar(16) * NumTraits<Scalar>::epsilon() * C_ref.norm();
|
||||
VERIFY((C - C_ref).norm() < tol);
|
||||
}
|
||||
|
||||
// ---- GEMM chaining: C = (A * B) then D = C * E -----------------------------
|
||||
|
||||
template <typename Scalar>
|
||||
void test_gemm_chain(Index n) {
|
||||
using Mat = Matrix<Scalar, Dynamic, Dynamic>;
|
||||
using RealScalar = typename NumTraits<Scalar>::Real;
|
||||
|
||||
Mat A = Mat::Random(n, n);
|
||||
Mat B = Mat::Random(n, n);
|
||||
Mat E = Mat::Random(n, n);
|
||||
|
||||
auto d_A = DeviceMatrix<Scalar>::fromHost(A);
|
||||
auto d_B = DeviceMatrix<Scalar>::fromHost(B);
|
||||
auto d_E = DeviceMatrix<Scalar>::fromHost(E);
|
||||
|
||||
DeviceMatrix<Scalar> d_C;
|
||||
d_C = d_A * d_B;
|
||||
DeviceMatrix<Scalar> d_D;
|
||||
d_D = d_C * d_E;
|
||||
|
||||
Mat D = d_D.toHost();
|
||||
Mat D_ref = (A * B) * E;
|
||||
|
||||
RealScalar tol = RealScalar(2) * RealScalar(n) * NumTraits<Scalar>::epsilon() * D_ref.norm();
|
||||
VERIFY((D - D_ref).norm() < tol);
|
||||
}
|
||||
|
||||
// ---- Square identity check: A * I = A ---------------------------------------
|
||||
|
||||
template <typename Scalar>
|
||||
void test_gemm_identity(Index n) {
|
||||
using Mat = Matrix<Scalar, Dynamic, Dynamic>;
|
||||
|
||||
Mat A = Mat::Random(n, n);
|
||||
Mat eye = Mat::Identity(n, n);
|
||||
|
||||
auto d_A = DeviceMatrix<Scalar>::fromHost(A);
|
||||
auto d_I = DeviceMatrix<Scalar>::fromHost(eye);
|
||||
|
||||
DeviceMatrix<Scalar> d_C;
|
||||
d_C = d_A * d_I;
|
||||
|
||||
Mat C = d_C.toHost();
|
||||
VERIFY_IS_APPROX(C, A);
|
||||
}
|
||||
|
||||
// ---- LLT solve expression: d_X = d_A.llt().solve(d_B) ----------------------
|
||||
|
||||
template <typename MatrixType>
|
||||
MatrixType make_spd(Index n) {
|
||||
using Scalar = typename MatrixType::Scalar;
|
||||
MatrixType M = MatrixType::Random(n, n);
|
||||
return M.adjoint() * M + MatrixType::Identity(n, n) * static_cast<Scalar>(n);
|
||||
}
|
||||
|
||||
template <typename Scalar>
|
||||
void test_llt_solve_expr(Index n, Index nrhs) {
|
||||
using Mat = Matrix<Scalar, Dynamic, Dynamic>;
|
||||
using RealScalar = typename NumTraits<Scalar>::Real;
|
||||
|
||||
Mat A = make_spd<Mat>(n);
|
||||
Mat B = Mat::Random(n, nrhs);
|
||||
|
||||
auto d_A = DeviceMatrix<Scalar>::fromHost(A);
|
||||
auto d_B = DeviceMatrix<Scalar>::fromHost(B);
|
||||
|
||||
DeviceMatrix<Scalar> d_X;
|
||||
d_X = d_A.llt().solve(d_B);
|
||||
|
||||
Mat X = d_X.toHost();
|
||||
RealScalar residual = (A * X - B).norm() / B.norm();
|
||||
VERIFY(residual < RealScalar(n) * NumTraits<Scalar>::epsilon());
|
||||
}
|
||||
|
||||
// ---- LLT solve with explicit context ----------------------------------------
|
||||
|
||||
template <typename Scalar>
|
||||
void test_llt_solve_expr_context(Index n, Index nrhs) {
|
||||
using Mat = Matrix<Scalar, Dynamic, Dynamic>;
|
||||
using RealScalar = typename NumTraits<Scalar>::Real;
|
||||
|
||||
Mat A = make_spd<Mat>(n);
|
||||
Mat B = Mat::Random(n, nrhs);
|
||||
|
||||
auto d_A = DeviceMatrix<Scalar>::fromHost(A);
|
||||
auto d_B = DeviceMatrix<Scalar>::fromHost(B);
|
||||
|
||||
GpuContext ctx;
|
||||
DeviceMatrix<Scalar> d_X;
|
||||
d_X.device(ctx) = d_A.llt().solve(d_B);
|
||||
|
||||
Mat X = d_X.toHost();
|
||||
RealScalar residual = (A * X - B).norm() / B.norm();
|
||||
VERIFY(residual < RealScalar(n) * NumTraits<Scalar>::epsilon());
|
||||
}
|
||||
|
||||
// ---- LU solve expression: d_X = d_A.lu().solve(d_B) ------------------------
|
||||
|
||||
template <typename Scalar>
|
||||
void test_lu_solve_expr(Index n, Index nrhs) {
|
||||
using Mat = Matrix<Scalar, Dynamic, Dynamic>;
|
||||
using RealScalar = typename NumTraits<Scalar>::Real;
|
||||
|
||||
Mat A = Mat::Random(n, n);
|
||||
Mat B = Mat::Random(n, nrhs);
|
||||
|
||||
auto d_A = DeviceMatrix<Scalar>::fromHost(A);
|
||||
auto d_B = DeviceMatrix<Scalar>::fromHost(B);
|
||||
|
||||
DeviceMatrix<Scalar> d_X;
|
||||
d_X = d_A.lu().solve(d_B);
|
||||
|
||||
Mat X = d_X.toHost();
|
||||
RealScalar residual = (A * X - B).norm() / (A.norm() * X.norm());
|
||||
VERIFY(residual < RealScalar(10) * RealScalar(n) * NumTraits<Scalar>::epsilon());
|
||||
}
|
||||
|
||||
// ---- GEMM + solver chain: C = A * B, X = C.llt().solve(D) ------------------
|
||||
|
||||
template <typename Scalar>
|
||||
void test_gemm_then_solve(Index n) {
|
||||
using Mat = Matrix<Scalar, Dynamic, Dynamic>;
|
||||
using RealScalar = typename NumTraits<Scalar>::Real;
|
||||
|
||||
Mat A = Mat::Random(n, n);
|
||||
Mat D = Mat::Random(n, 1);
|
||||
|
||||
// Make SPD: C = A^H * A + n*I
|
||||
auto d_A = DeviceMatrix<Scalar>::fromHost(A);
|
||||
DeviceMatrix<Scalar> d_C;
|
||||
d_C = d_A.adjoint() * d_A;
|
||||
|
||||
// Add n*I on host (no element-wise ops on DeviceMatrix yet).
|
||||
Mat C = d_C.toHost();
|
||||
C += Mat::Identity(n, n) * static_cast<Scalar>(n);
|
||||
d_C = DeviceMatrix<Scalar>::fromHost(C);
|
||||
|
||||
auto d_D = DeviceMatrix<Scalar>::fromHost(D);
|
||||
|
||||
DeviceMatrix<Scalar> d_X;
|
||||
d_X = d_C.llt().solve(d_D);
|
||||
|
||||
Mat X = d_X.toHost();
|
||||
RealScalar residual = (C * X - D).norm() / D.norm();
|
||||
VERIFY(residual < RealScalar(n) * NumTraits<Scalar>::epsilon());
|
||||
}
|
||||
|
||||
// ---- LLT solve with Upper triangle -----------------------------------------
|
||||
|
||||
template <typename Scalar>
|
||||
void test_llt_solve_upper(Index n, Index nrhs) {
|
||||
using Mat = Matrix<Scalar, Dynamic, Dynamic>;
|
||||
using RealScalar = typename NumTraits<Scalar>::Real;
|
||||
|
||||
Mat A = make_spd<Mat>(n);
|
||||
Mat B = Mat::Random(n, nrhs);
|
||||
|
||||
auto d_A = DeviceMatrix<Scalar>::fromHost(A);
|
||||
auto d_B = DeviceMatrix<Scalar>::fromHost(B);
|
||||
|
||||
DeviceMatrix<Scalar> d_X;
|
||||
d_X = d_A.template llt<Upper>().solve(d_B);
|
||||
|
||||
Mat X = d_X.toHost();
|
||||
RealScalar residual = (A * X - B).norm() / B.norm();
|
||||
VERIFY(residual < RealScalar(n) * NumTraits<Scalar>::epsilon());
|
||||
}
|
||||
|
||||
// ---- LU solve with explicit context -----------------------------------------
|
||||
|
||||
template <typename Scalar>
|
||||
void test_lu_solve_expr_context(Index n, Index nrhs) {
|
||||
using Mat = Matrix<Scalar, Dynamic, Dynamic>;
|
||||
using RealScalar = typename NumTraits<Scalar>::Real;
|
||||
|
||||
Mat A = Mat::Random(n, n);
|
||||
Mat B = Mat::Random(n, nrhs);
|
||||
|
||||
auto d_A = DeviceMatrix<Scalar>::fromHost(A);
|
||||
auto d_B = DeviceMatrix<Scalar>::fromHost(B);
|
||||
|
||||
GpuContext ctx;
|
||||
DeviceMatrix<Scalar> d_X;
|
||||
d_X.device(ctx) = d_A.lu().solve(d_B);
|
||||
|
||||
Mat X = d_X.toHost();
|
||||
RealScalar residual = (A * X - B).norm() / (A.norm() * X.norm());
|
||||
VERIFY(residual < RealScalar(10) * RealScalar(n) * NumTraits<Scalar>::epsilon());
|
||||
}
|
||||
|
||||
// ---- Zero-nrhs solver expressions ------------------------------------------
|
||||
|
||||
template <typename Scalar>
|
||||
void test_llt_solve_zero_nrhs(Index n) {
|
||||
using Mat = Matrix<Scalar, Dynamic, Dynamic>;
|
||||
|
||||
Mat A = make_spd<Mat>(n);
|
||||
Mat B = Mat::Random(n, 0);
|
||||
|
||||
auto d_A = DeviceMatrix<Scalar>::fromHost(A);
|
||||
auto d_B = DeviceMatrix<Scalar>::fromHost(B);
|
||||
|
||||
DeviceMatrix<Scalar> d_X;
|
||||
d_X = d_A.llt().solve(d_B);
|
||||
|
||||
VERIFY_IS_EQUAL(d_X.rows(), n);
|
||||
VERIFY_IS_EQUAL(d_X.cols(), 0);
|
||||
}
|
||||
|
||||
template <typename Scalar>
|
||||
void test_lu_solve_zero_nrhs(Index n) {
|
||||
using Mat = Matrix<Scalar, Dynamic, Dynamic>;
|
||||
|
||||
Mat A = Mat::Random(n, n);
|
||||
Mat B = Mat::Random(n, 0);
|
||||
|
||||
auto d_A = DeviceMatrix<Scalar>::fromHost(A);
|
||||
auto d_B = DeviceMatrix<Scalar>::fromHost(B);
|
||||
|
||||
DeviceMatrix<Scalar> d_X;
|
||||
d_X = d_A.lu().solve(d_B);
|
||||
|
||||
VERIFY_IS_EQUAL(d_X.rows(), n);
|
||||
VERIFY_IS_EQUAL(d_X.cols(), 0);
|
||||
}
|
||||
|
||||
// ---- TRSM: triangularView<UpLo>().solve(B) ----------------------------------
|
||||
|
||||
template <typename Scalar, int UpLo>
|
||||
void test_trsm(Index n, Index nrhs) {
|
||||
using Mat = Matrix<Scalar, Dynamic, Dynamic>;
|
||||
using RealScalar = typename NumTraits<Scalar>::Real;
|
||||
|
||||
// Build a well-conditioned triangular matrix.
|
||||
Mat A = Mat::Random(n, n);
|
||||
A.diagonal().array() += static_cast<Scalar>(n); // ensure non-singular
|
||||
if (UpLo == Lower)
|
||||
A = A.template triangularView<Lower>();
|
||||
else
|
||||
A = A.template triangularView<Upper>();
|
||||
|
||||
Mat B = Mat::Random(n, nrhs);
|
||||
|
||||
auto d_A = DeviceMatrix<Scalar>::fromHost(A);
|
||||
auto d_B = DeviceMatrix<Scalar>::fromHost(B);
|
||||
|
||||
DeviceMatrix<Scalar> d_X;
|
||||
d_X = d_A.template triangularView<UpLo>().solve(d_B);
|
||||
|
||||
Mat X = d_X.toHost();
|
||||
RealScalar residual = (A * X - B).norm() / B.norm();
|
||||
VERIFY(residual < RealScalar(n) * NumTraits<Scalar>::epsilon());
|
||||
}
|
||||
|
||||
// ---- SYMM/HEMM: selfadjointView<UpLo>() * B --------------------------------
|
||||
|
||||
template <typename Scalar, int UpLo>
|
||||
void test_symm(Index n, Index nrhs) {
|
||||
using Mat = Matrix<Scalar, Dynamic, Dynamic>;
|
||||
using RealScalar = typename NumTraits<Scalar>::Real;
|
||||
|
||||
Mat A = make_spd<Mat>(n); // SPD is also self-adjoint
|
||||
Mat B = Mat::Random(n, nrhs);
|
||||
|
||||
auto d_A = DeviceMatrix<Scalar>::fromHost(A);
|
||||
auto d_B = DeviceMatrix<Scalar>::fromHost(B);
|
||||
|
||||
DeviceMatrix<Scalar> d_C;
|
||||
d_C = d_A.template selfadjointView<UpLo>() * d_B;
|
||||
|
||||
Mat C = d_C.toHost();
|
||||
Mat C_ref = A * B; // A is symmetric, so full multiply == symm
|
||||
|
||||
RealScalar tol = RealScalar(n) * NumTraits<Scalar>::epsilon() * C_ref.norm();
|
||||
VERIFY((C - C_ref).norm() < tol);
|
||||
}
|
||||
|
||||
// ---- SYRK/HERK: rankUpdate(A) → C = A * A^H --------------------------------
|
||||
|
||||
template <typename Scalar>
|
||||
void test_syrk(Index n, Index k) {
|
||||
using Mat = Matrix<Scalar, Dynamic, Dynamic>;
|
||||
using RealScalar = typename NumTraits<Scalar>::Real;
|
||||
|
||||
Mat A = Mat::Random(n, k);
|
||||
|
||||
auto d_A = DeviceMatrix<Scalar>::fromHost(A);
|
||||
|
||||
DeviceMatrix<Scalar> d_C;
|
||||
d_C.template selfadjointView<Lower>().rankUpdate(d_A);
|
||||
|
||||
Mat C = d_C.toHost();
|
||||
// Only lower triangle is meaningful for SYRK. Compare lower triangle.
|
||||
Mat C_ref = A * A.adjoint();
|
||||
|
||||
// Extract lower triangle for comparison.
|
||||
Mat C_lower = C.template triangularView<Lower>();
|
||||
Mat C_ref_lower = C_ref.template triangularView<Lower>();
|
||||
|
||||
RealScalar tol = RealScalar(k) * NumTraits<Scalar>::epsilon() * C_ref.norm();
|
||||
VERIFY((C_lower - C_ref_lower).norm() < tol);
|
||||
}
|
||||
|
||||
// ---- Per-scalar driver ------------------------------------------------------
|
||||
|
||||
template <typename Scalar>
|
||||
void test_scalar() {
|
||||
CALL_SUBTEST(test_gemm_basic<Scalar>(64, 64, 64));
|
||||
CALL_SUBTEST(test_gemm_basic<Scalar>(128, 64, 32));
|
||||
CALL_SUBTEST(test_gemm_basic<Scalar>(1, 1, 1));
|
||||
CALL_SUBTEST(test_gemm_basic<Scalar>(256, 256, 256));
|
||||
|
||||
CALL_SUBTEST(test_gemm_adjoint_lhs<Scalar>(64, 64, 64));
|
||||
CALL_SUBTEST(test_gemm_adjoint_lhs<Scalar>(128, 32, 64));
|
||||
|
||||
CALL_SUBTEST(test_gemm_transpose_rhs<Scalar>(64, 64, 64));
|
||||
CALL_SUBTEST(test_gemm_transpose_rhs<Scalar>(128, 32, 64));
|
||||
|
||||
CALL_SUBTEST(test_gemm_scaled<Scalar>(64, 64, 64));
|
||||
CALL_SUBTEST(test_gemm_scaled_rhs<Scalar>(64, 64, 64));
|
||||
CALL_SUBTEST(test_gemm_accumulate<Scalar>(64, 64, 64));
|
||||
CALL_SUBTEST(test_gemm_accumulate_empty<Scalar>(64, 64, 64));
|
||||
CALL_SUBTEST(test_gemm_subtract<Scalar>(64, 64, 64));
|
||||
CALL_SUBTEST(test_gemm_subtract_empty<Scalar>(64, 64, 64));
|
||||
CALL_SUBTEST(test_gemm_dimension_mismatch<Scalar>());
|
||||
CALL_SUBTEST(test_gemm_explicit_context<Scalar>(64, 64, 64));
|
||||
CALL_SUBTEST(test_gemm_cross_context_reuse<Scalar>(64));
|
||||
CALL_SUBTEST(test_gemm_cross_context_resize<Scalar>());
|
||||
CALL_SUBTEST(test_gemm_chain<Scalar>(64));
|
||||
CALL_SUBTEST(test_gemm_identity<Scalar>(64));
|
||||
|
||||
// Solver expressions — zero-size edge cases (use dedicated tests, not residual-based)
|
||||
|
||||
// Solver expressions
|
||||
CALL_SUBTEST(test_llt_solve_expr<Scalar>(64, 1));
|
||||
CALL_SUBTEST(test_llt_solve_expr<Scalar>(64, 4));
|
||||
CALL_SUBTEST(test_llt_solve_expr<Scalar>(256, 8));
|
||||
CALL_SUBTEST(test_llt_solve_expr_context<Scalar>(64, 4));
|
||||
CALL_SUBTEST(test_llt_solve_upper<Scalar>(64, 4));
|
||||
CALL_SUBTEST(test_lu_solve_expr<Scalar>(64, 1));
|
||||
CALL_SUBTEST(test_lu_solve_expr<Scalar>(64, 4));
|
||||
CALL_SUBTEST(test_lu_solve_expr<Scalar>(256, 8));
|
||||
CALL_SUBTEST(test_lu_solve_expr_context<Scalar>(64, 4));
|
||||
CALL_SUBTEST(test_llt_solve_zero_nrhs<Scalar>(64));
|
||||
CALL_SUBTEST(test_llt_solve_zero_nrhs<Scalar>(0));
|
||||
CALL_SUBTEST(test_lu_solve_zero_nrhs<Scalar>(64));
|
||||
CALL_SUBTEST(test_lu_solve_zero_nrhs<Scalar>(0));
|
||||
CALL_SUBTEST(test_gemm_then_solve<Scalar>(64));
|
||||
|
||||
// TRSM
|
||||
CALL_SUBTEST((test_trsm<Scalar, Lower>(64, 1)));
|
||||
CALL_SUBTEST((test_trsm<Scalar, Lower>(64, 4)));
|
||||
CALL_SUBTEST((test_trsm<Scalar, Upper>(64, 4)));
|
||||
CALL_SUBTEST((test_trsm<Scalar, Lower>(256, 8)));
|
||||
|
||||
// SYMM/HEMM
|
||||
CALL_SUBTEST((test_symm<Scalar, Lower>(64, 4)));
|
||||
CALL_SUBTEST((test_symm<Scalar, Upper>(64, 4)));
|
||||
CALL_SUBTEST((test_symm<Scalar, Lower>(128, 8)));
|
||||
|
||||
// SYRK/HERK
|
||||
CALL_SUBTEST(test_syrk<Scalar>(64, 64));
|
||||
CALL_SUBTEST(test_syrk<Scalar>(64, 32));
|
||||
CALL_SUBTEST(test_syrk<Scalar>(128, 64));
|
||||
}
|
||||
|
||||
// ---- Solver failure mode tests (not templated on Scalar) --------------------
|
||||
|
||||
void test_llt_not_spd() {
|
||||
// Negative definite matrix — LLT factorization must fail.
|
||||
MatrixXd A = -MatrixXd::Identity(8, 8);
|
||||
MatrixXd B = MatrixXd::Random(8, 1);
|
||||
auto d_A = DeviceMatrix<double>::fromHost(A);
|
||||
auto d_B = DeviceMatrix<double>::fromHost(B);
|
||||
DeviceMatrix<double> d_X;
|
||||
VERIFY_RAISES_ASSERT(d_X = d_A.llt().solve(d_B));
|
||||
}
|
||||
|
||||
void test_lu_singular() {
|
||||
// Zero matrix — LU factorization must detect singularity.
|
||||
MatrixXd A = MatrixXd::Zero(8, 8);
|
||||
MatrixXd B = MatrixXd::Random(8, 1);
|
||||
auto d_A = DeviceMatrix<double>::fromHost(A);
|
||||
auto d_B = DeviceMatrix<double>::fromHost(B);
|
||||
DeviceMatrix<double> d_X;
|
||||
VERIFY_RAISES_ASSERT(d_X = d_A.lu().solve(d_B));
|
||||
}
|
||||
|
||||
EIGEN_DECLARE_TEST(gpu_cublas) {
|
||||
CALL_SUBTEST(test_scalar<float>());
|
||||
CALL_SUBTEST(test_scalar<double>());
|
||||
CALL_SUBTEST(test_scalar<std::complex<float>>());
|
||||
CALL_SUBTEST(test_scalar<std::complex<double>>());
|
||||
CALL_SUBTEST(test_llt_not_spd());
|
||||
CALL_SUBTEST(test_lu_singular());
|
||||
}
|
||||
210
test/gpu_cusolver_llt.cpp
Normal file
210
test/gpu_cusolver_llt.cpp
Normal file
@@ -0,0 +1,210 @@
|
||||
// This file is part of Eigen, a lightweight C++ template library
|
||||
// for linear algebra.
|
||||
//
|
||||
// Copyright (C) 2026 Eigen Authors
|
||||
//
|
||||
// This Source Code Form is subject to the terms of the Mozilla
|
||||
// Public License v. 2.0. If a copy of the MPL was not distributed
|
||||
// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
|
||||
// Tests for GpuLLT: GPU Cholesky (LL^T) using cuSOLVER.
|
||||
// Covers cusolverDnXpotrf (factorization) and cusolverDnXpotrs (solve)
|
||||
// for float, double, complex<float>, complex<double>, Lower and Upper.
|
||||
|
||||
#define EIGEN_USE_GPU
|
||||
#include "main.h"
|
||||
#include <Eigen/Cholesky>
|
||||
#include <Eigen/GPU>
|
||||
|
||||
using namespace Eigen;
|
||||
|
||||
// Build a random symmetric positive-definite matrix: A = M^H*M + n*I.
|
||||
template <typename MatrixType>
|
||||
MatrixType make_spd(Index n) {
|
||||
using Scalar = typename MatrixType::Scalar;
|
||||
MatrixType M = MatrixType::Random(n, n);
|
||||
return M.adjoint() * M + MatrixType::Identity(n, n) * static_cast<Scalar>(n);
|
||||
}
|
||||
|
||||
// Test factorization: L*L^H must reconstruct A to within floating-point tolerance.
|
||||
template <typename Scalar, int UpLo>
|
||||
void test_potrf(Index n) {
|
||||
using MatrixType = Matrix<Scalar, Dynamic, Dynamic>;
|
||||
using RealScalar = typename NumTraits<Scalar>::Real;
|
||||
|
||||
MatrixType A = make_spd<MatrixType>(n);
|
||||
|
||||
GpuLLT<Scalar, UpLo> llt(A);
|
||||
VERIFY_IS_EQUAL(llt.info(), Success);
|
||||
|
||||
// Reconstruct L*L^H and compare to original A.
|
||||
// GpuLLT stores the factor on device; use CPU LLT to get the triangular factor
|
||||
// for reconstruction since GpuLLT does not expose the device-resident factor directly.
|
||||
LLT<MatrixType, UpLo> ref(A);
|
||||
VERIFY_IS_EQUAL(ref.info(), Success);
|
||||
MatrixType A_reconstructed = ref.reconstructedMatrix();
|
||||
|
||||
// Both should equal A to within n*eps*||A||.
|
||||
RealScalar tol = RealScalar(4) * RealScalar(n) * NumTraits<Scalar>::epsilon() * A.norm();
|
||||
VERIFY((A_reconstructed - A).norm() < tol);
|
||||
|
||||
// Smoke-test: llt.solve(b) should return the same result as ref.solve(b).
|
||||
MatrixType b = MatrixType::Random(n, 1);
|
||||
MatrixType x_gpu = llt.solve(b);
|
||||
MatrixType x_cpu = ref.solve(b);
|
||||
VERIFY((x_gpu - x_cpu).norm() < tol);
|
||||
}
|
||||
|
||||
// Test solve: residual ||A*X - B|| / ||B|| must be small.
|
||||
template <typename Scalar, int UpLo>
|
||||
void test_potrs(Index n, Index nrhs) {
|
||||
using MatrixType = Matrix<Scalar, Dynamic, Dynamic>;
|
||||
using RealScalar = typename NumTraits<Scalar>::Real;
|
||||
|
||||
MatrixType A = make_spd<MatrixType>(n);
|
||||
MatrixType B = MatrixType::Random(n, nrhs);
|
||||
|
||||
GpuLLT<Scalar, UpLo> llt(A);
|
||||
VERIFY_IS_EQUAL(llt.info(), Success);
|
||||
|
||||
MatrixType X = llt.solve(B);
|
||||
|
||||
RealScalar residual = (A * X - B).norm() / B.norm();
|
||||
RealScalar tol = RealScalar(n) * NumTraits<Scalar>::epsilon();
|
||||
VERIFY(residual < tol);
|
||||
}
|
||||
|
||||
// Test that multiple solves against the same factor all produce correct results.
|
||||
// This exercises the key design property: L stays on device across calls.
|
||||
template <typename Scalar>
|
||||
void test_multiple_solves(Index n) {
|
||||
using MatrixType = Matrix<Scalar, Dynamic, Dynamic>;
|
||||
using RealScalar = typename NumTraits<Scalar>::Real;
|
||||
|
||||
MatrixType A = make_spd<MatrixType>(n);
|
||||
GpuLLT<Scalar, Lower> llt(A);
|
||||
VERIFY_IS_EQUAL(llt.info(), Success);
|
||||
|
||||
RealScalar tol = RealScalar(n) * NumTraits<Scalar>::epsilon();
|
||||
for (int k = 0; k < 5; ++k) {
|
||||
MatrixType B = MatrixType::Random(n, 3);
|
||||
MatrixType X = llt.solve(B);
|
||||
RealScalar residual = (A * X - B).norm() / B.norm();
|
||||
VERIFY(residual < tol);
|
||||
}
|
||||
}
|
||||
|
||||
// Test that GpuLLT correctly detects a non-SPD matrix.
|
||||
void test_not_spd() {
|
||||
MatrixXd A = -MatrixXd::Identity(8, 8); // negative definite
|
||||
GpuLLT<double> llt(A);
|
||||
VERIFY_IS_EQUAL(llt.info(), NumericalIssue);
|
||||
}
|
||||
|
||||
// ---- DeviceMatrix integration tests -----------------------------------------
|
||||
|
||||
// compute(DeviceMatrix) + solve(DeviceMatrix) → toHost
|
||||
template <typename Scalar, int UpLo>
|
||||
void test_device_matrix_solve(Index n, Index nrhs) {
|
||||
using MatrixType = Matrix<Scalar, Dynamic, Dynamic>;
|
||||
using RealScalar = typename NumTraits<Scalar>::Real;
|
||||
|
||||
MatrixType A = make_spd<MatrixType>(n);
|
||||
MatrixType B = MatrixType::Random(n, nrhs);
|
||||
|
||||
auto d_A = DeviceMatrix<Scalar>::fromHost(A);
|
||||
auto d_B = DeviceMatrix<Scalar>::fromHost(B);
|
||||
|
||||
GpuLLT<Scalar, UpLo> llt;
|
||||
llt.compute(d_A);
|
||||
VERIFY_IS_EQUAL(llt.info(), Success);
|
||||
|
||||
DeviceMatrix<Scalar> d_X = llt.solve(d_B);
|
||||
MatrixType X = d_X.toHost();
|
||||
|
||||
RealScalar residual = (A * X - B).norm() / B.norm();
|
||||
VERIFY(residual < RealScalar(n) * NumTraits<Scalar>::epsilon());
|
||||
}
|
||||
|
||||
// compute(DeviceMatrix&&) — move path
|
||||
template <typename Scalar>
|
||||
void test_device_matrix_move_compute(Index n) {
|
||||
using MatrixType = Matrix<Scalar, Dynamic, Dynamic>;
|
||||
using RealScalar = typename NumTraits<Scalar>::Real;
|
||||
|
||||
MatrixType A = make_spd<MatrixType>(n);
|
||||
MatrixType B = MatrixType::Random(n, 1);
|
||||
|
||||
auto d_A = DeviceMatrix<Scalar>::fromHost(A);
|
||||
GpuLLT<Scalar, Lower> llt;
|
||||
llt.compute(std::move(d_A));
|
||||
VERIFY_IS_EQUAL(llt.info(), Success);
|
||||
|
||||
// d_A should be empty after move.
|
||||
VERIFY(d_A.empty());
|
||||
|
||||
MatrixType X = llt.solve(B);
|
||||
RealScalar residual = (A * X - B).norm() / B.norm();
|
||||
VERIFY(residual < RealScalar(n) * NumTraits<Scalar>::epsilon());
|
||||
}
|
||||
|
||||
// Full async chain: compute → solve → solve again with result as RHS → toHost
|
||||
template <typename Scalar>
|
||||
void test_chaining(Index n) {
|
||||
using MatrixType = Matrix<Scalar, Dynamic, Dynamic>;
|
||||
using RealScalar = typename NumTraits<Scalar>::Real;
|
||||
|
||||
MatrixType A = make_spd<MatrixType>(n);
|
||||
MatrixType B = MatrixType::Random(n, 3);
|
||||
|
||||
auto d_A = DeviceMatrix<Scalar>::fromHost(A);
|
||||
auto d_B = DeviceMatrix<Scalar>::fromHost(B);
|
||||
|
||||
GpuLLT<Scalar, Lower> llt;
|
||||
llt.compute(d_A);
|
||||
VERIFY_IS_EQUAL(llt.info(), Success);
|
||||
|
||||
// Chain: solve → use result as RHS for another solve
|
||||
DeviceMatrix<Scalar> d_X = llt.solve(d_B);
|
||||
DeviceMatrix<Scalar> d_Y = llt.solve(d_X);
|
||||
|
||||
// Only sync at the very end.
|
||||
MatrixType Y = d_Y.toHost();
|
||||
|
||||
// Verify: Y = A^{-2} * B
|
||||
MatrixType X_ref = LLT<MatrixType, Lower>(A).solve(B);
|
||||
MatrixType Y_ref = LLT<MatrixType, Lower>(A).solve(X_ref);
|
||||
|
||||
RealScalar tol = RealScalar(4) * RealScalar(n) * NumTraits<Scalar>::epsilon() * Y_ref.norm();
|
||||
VERIFY((Y - Y_ref).norm() < tol);
|
||||
}
|
||||
|
||||
template <typename Scalar>
|
||||
void test_scalar() {
|
||||
CALL_SUBTEST((test_potrf<Scalar, Lower>(1)));
|
||||
CALL_SUBTEST((test_potrf<Scalar, Lower>(64)));
|
||||
CALL_SUBTEST((test_potrf<Scalar, Lower>(256)));
|
||||
CALL_SUBTEST((test_potrf<Scalar, Upper>(64)));
|
||||
CALL_SUBTEST((test_potrf<Scalar, Upper>(256)));
|
||||
|
||||
CALL_SUBTEST((test_potrs<Scalar, Lower>(64, 1)));
|
||||
CALL_SUBTEST((test_potrs<Scalar, Lower>(64, 4)));
|
||||
CALL_SUBTEST((test_potrs<Scalar, Lower>(256, 8)));
|
||||
CALL_SUBTEST((test_potrs<Scalar, Upper>(64, 1)));
|
||||
CALL_SUBTEST((test_potrs<Scalar, Upper>(256, 4)));
|
||||
|
||||
CALL_SUBTEST(test_multiple_solves<Scalar>(128));
|
||||
|
||||
CALL_SUBTEST((test_device_matrix_solve<Scalar, Lower>(64, 4)));
|
||||
CALL_SUBTEST((test_device_matrix_solve<Scalar, Upper>(128, 1)));
|
||||
CALL_SUBTEST(test_device_matrix_move_compute<Scalar>(64));
|
||||
CALL_SUBTEST(test_chaining<Scalar>(64));
|
||||
}
|
||||
|
||||
EIGEN_DECLARE_TEST(gpu_cusolver_llt) {
|
||||
CALL_SUBTEST(test_scalar<float>());
|
||||
CALL_SUBTEST(test_scalar<double>());
|
||||
CALL_SUBTEST(test_scalar<std::complex<float>>());
|
||||
CALL_SUBTEST(test_scalar<std::complex<double>>());
|
||||
CALL_SUBTEST(test_not_spd());
|
||||
}
|
||||
206
test/gpu_cusolver_lu.cpp
Normal file
206
test/gpu_cusolver_lu.cpp
Normal file
@@ -0,0 +1,206 @@
|
||||
// This file is part of Eigen, a lightweight C++ template library
|
||||
// for linear algebra.
|
||||
//
|
||||
// Copyright (C) 2026 Eigen Authors
|
||||
//
|
||||
// This Source Code Form is subject to the terms of the Mozilla
|
||||
// Public License v. 2.0. If a copy of the MPL was not distributed
|
||||
// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
|
||||
// Tests for GpuLU: GPU partial-pivoting LU decomposition via cuSOLVER.
|
||||
// Covers cusolverDnXgetrf (factorization) and cusolverDnXgetrs (solve)
|
||||
// for float, double, complex<float>, complex<double>.
|
||||
//
|
||||
#define EIGEN_USE_GPU
|
||||
#include "main.h"
|
||||
#include <Eigen/LU>
|
||||
#include <Eigen/GPU>
|
||||
|
||||
using namespace Eigen;
|
||||
|
||||
// ---- Test factorization + NoTrans solve: residual ||A*X - B|| / ||B|| -------
|
||||
|
||||
template <typename Scalar>
|
||||
void test_getrf(Index n) {
|
||||
using MatrixType = Matrix<Scalar, Dynamic, Dynamic>;
|
||||
using RealScalar = typename NumTraits<Scalar>::Real;
|
||||
|
||||
MatrixType A = MatrixType::Random(n, n);
|
||||
MatrixType B = MatrixType::Random(n, 4);
|
||||
|
||||
GpuLU<Scalar> lu(A);
|
||||
VERIFY_IS_EQUAL(lu.info(), Success);
|
||||
|
||||
MatrixType X = lu.solve(B);
|
||||
// Backward error bound for LU: ||A*X - B|| <= O(n*u) * ||A|| * ||X||.
|
||||
// Normalize by ||A||*||X|| rather than ||B|| to be condition-number agnostic.
|
||||
RealScalar residual = (A * X - B).norm() / (A.norm() * X.norm());
|
||||
VERIFY(residual < RealScalar(10) * RealScalar(n) * NumTraits<Scalar>::epsilon());
|
||||
}
|
||||
|
||||
// ---- Test solve: A^T*X = B and A^H*X = B ------------------------------------
|
||||
|
||||
template <typename Scalar>
|
||||
void test_getrs_trans(Index n) {
|
||||
using MatrixType = Matrix<Scalar, Dynamic, Dynamic>;
|
||||
using RealScalar = typename NumTraits<Scalar>::Real;
|
||||
|
||||
MatrixType A = MatrixType::Random(n, n);
|
||||
MatrixType B = MatrixType::Random(n, 3);
|
||||
RealScalar tol = RealScalar(10) * RealScalar(n) * NumTraits<Scalar>::epsilon();
|
||||
|
||||
GpuLU<Scalar> lu(A);
|
||||
VERIFY_IS_EQUAL(lu.info(), Success);
|
||||
|
||||
MatrixType Xt = lu.solve(B, GpuLU<Scalar>::Transpose);
|
||||
VERIFY((A.transpose() * Xt - B).norm() / (A.norm() * Xt.norm()) < tol);
|
||||
|
||||
MatrixType Xc = lu.solve(B, GpuLU<Scalar>::ConjugateTranspose);
|
||||
VERIFY((A.adjoint() * Xc - B).norm() / (A.norm() * Xc.norm()) < tol);
|
||||
}
|
||||
|
||||
// ---- Test multiple solves reuse the device-resident LU ----------------------
|
||||
|
||||
template <typename Scalar>
|
||||
void test_multiple_solves(Index n) {
|
||||
using MatrixType = Matrix<Scalar, Dynamic, Dynamic>;
|
||||
using RealScalar = typename NumTraits<Scalar>::Real;
|
||||
|
||||
MatrixType A = MatrixType::Random(n, n);
|
||||
GpuLU<Scalar> lu(A);
|
||||
VERIFY_IS_EQUAL(lu.info(), Success);
|
||||
|
||||
RealScalar tol = RealScalar(10) * RealScalar(n) * NumTraits<Scalar>::epsilon();
|
||||
for (int k = 0; k < 5; ++k) {
|
||||
MatrixType B = MatrixType::Random(n, 3);
|
||||
MatrixType X = lu.solve(B);
|
||||
VERIFY((A * X - B).norm() / (A.norm() * X.norm()) < tol);
|
||||
}
|
||||
}
|
||||
|
||||
// ---- Agreement with CPU PartialPivLU ----------------------------------------
|
||||
|
||||
template <typename Scalar>
|
||||
void test_vs_cpu(Index n) {
|
||||
using MatrixType = Matrix<Scalar, Dynamic, Dynamic>;
|
||||
using RealScalar = typename NumTraits<Scalar>::Real;
|
||||
|
||||
MatrixType A = MatrixType::Random(n, n);
|
||||
MatrixType B = MatrixType::Random(n, 5);
|
||||
|
||||
GpuLU<Scalar> gpu_lu(A);
|
||||
VERIFY_IS_EQUAL(gpu_lu.info(), Success);
|
||||
|
||||
MatrixType X_gpu = gpu_lu.solve(B);
|
||||
MatrixType X_cpu = PartialPivLU<MatrixType>(A).solve(B);
|
||||
|
||||
RealScalar tol = RealScalar(100) * RealScalar(n) * NumTraits<Scalar>::epsilon();
|
||||
VERIFY((X_gpu - X_cpu).norm() / X_cpu.norm() < tol);
|
||||
}
|
||||
|
||||
// ---- Singular matrix detection ----------------------------------------------
|
||||
|
||||
void test_singular() {
|
||||
MatrixXd A = MatrixXd::Zero(8, 8);
|
||||
GpuLU<double> lu(A);
|
||||
VERIFY_IS_EQUAL(lu.info(), NumericalIssue);
|
||||
}
|
||||
|
||||
// ---- DeviceMatrix integration tests -----------------------------------------
|
||||
|
||||
template <typename Scalar>
|
||||
void test_device_matrix_solve(Index n) {
|
||||
using MatrixType = Matrix<Scalar, Dynamic, Dynamic>;
|
||||
using RealScalar = typename NumTraits<Scalar>::Real;
|
||||
|
||||
MatrixType A = MatrixType::Random(n, n);
|
||||
MatrixType B = MatrixType::Random(n, 4);
|
||||
|
||||
auto d_A = DeviceMatrix<Scalar>::fromHost(A);
|
||||
auto d_B = DeviceMatrix<Scalar>::fromHost(B);
|
||||
|
||||
GpuLU<Scalar> lu;
|
||||
lu.compute(d_A);
|
||||
VERIFY_IS_EQUAL(lu.info(), Success);
|
||||
|
||||
DeviceMatrix<Scalar> d_X = lu.solve(d_B);
|
||||
MatrixType X = d_X.toHost();
|
||||
|
||||
RealScalar residual = (A * X - B).norm() / (A.norm() * X.norm());
|
||||
VERIFY(residual < RealScalar(10) * RealScalar(n) * NumTraits<Scalar>::epsilon());
|
||||
}
|
||||
|
||||
template <typename Scalar>
|
||||
void test_device_matrix_move_compute(Index n) {
|
||||
using MatrixType = Matrix<Scalar, Dynamic, Dynamic>;
|
||||
using RealScalar = typename NumTraits<Scalar>::Real;
|
||||
|
||||
MatrixType A = MatrixType::Random(n, n);
|
||||
MatrixType B = MatrixType::Random(n, 1);
|
||||
|
||||
auto d_A = DeviceMatrix<Scalar>::fromHost(A);
|
||||
GpuLU<Scalar> lu;
|
||||
lu.compute(std::move(d_A));
|
||||
VERIFY_IS_EQUAL(lu.info(), Success);
|
||||
VERIFY(d_A.empty());
|
||||
|
||||
MatrixType X = lu.solve(B);
|
||||
RealScalar residual = (A * X - B).norm() / (A.norm() * X.norm());
|
||||
VERIFY(residual < RealScalar(10) * RealScalar(n) * NumTraits<Scalar>::epsilon());
|
||||
}
|
||||
|
||||
template <typename Scalar>
|
||||
void test_chaining(Index n) {
|
||||
using MatrixType = Matrix<Scalar, Dynamic, Dynamic>;
|
||||
using RealScalar = typename NumTraits<Scalar>::Real;
|
||||
|
||||
MatrixType A = MatrixType::Random(n, n);
|
||||
MatrixType B = MatrixType::Random(n, 3);
|
||||
|
||||
auto d_A = DeviceMatrix<Scalar>::fromHost(A);
|
||||
auto d_B = DeviceMatrix<Scalar>::fromHost(B);
|
||||
|
||||
GpuLU<Scalar> lu;
|
||||
lu.compute(d_A);
|
||||
VERIFY_IS_EQUAL(lu.info(), Success);
|
||||
|
||||
// Chain: solve → use result as RHS
|
||||
DeviceMatrix<Scalar> d_X = lu.solve(d_B);
|
||||
DeviceMatrix<Scalar> d_Y = lu.solve(d_X);
|
||||
MatrixType Y = d_Y.toHost();
|
||||
|
||||
MatrixType X_ref = PartialPivLU<MatrixType>(A).solve(B);
|
||||
MatrixType Y_ref = PartialPivLU<MatrixType>(A).solve(X_ref);
|
||||
|
||||
RealScalar tol = RealScalar(100) * RealScalar(n) * NumTraits<Scalar>::epsilon() * Y_ref.norm();
|
||||
VERIFY((Y - Y_ref).norm() < tol);
|
||||
}
|
||||
|
||||
// ---- Per-scalar driver -------------------------------------------------------
|
||||
|
||||
template <typename Scalar>
|
||||
void test_scalar() {
|
||||
CALL_SUBTEST(test_getrf<Scalar>(1));
|
||||
CALL_SUBTEST(test_getrf<Scalar>(64));
|
||||
CALL_SUBTEST(test_getrf<Scalar>(256));
|
||||
|
||||
CALL_SUBTEST(test_getrs_trans<Scalar>(64));
|
||||
CALL_SUBTEST(test_getrs_trans<Scalar>(128));
|
||||
|
||||
CALL_SUBTEST(test_multiple_solves<Scalar>(128));
|
||||
|
||||
CALL_SUBTEST(test_vs_cpu<Scalar>(64));
|
||||
CALL_SUBTEST(test_vs_cpu<Scalar>(256));
|
||||
|
||||
CALL_SUBTEST(test_device_matrix_solve<Scalar>(64));
|
||||
CALL_SUBTEST(test_device_matrix_move_compute<Scalar>(64));
|
||||
CALL_SUBTEST(test_chaining<Scalar>(64));
|
||||
}
|
||||
|
||||
EIGEN_DECLARE_TEST(gpu_cusolver_lu) {
|
||||
CALL_SUBTEST(test_scalar<float>());
|
||||
CALL_SUBTEST(test_scalar<double>());
|
||||
CALL_SUBTEST(test_scalar<std::complex<float>>());
|
||||
CALL_SUBTEST(test_scalar<std::complex<double>>());
|
||||
CALL_SUBTEST(test_singular());
|
||||
}
|
||||
247
test/gpu_device_matrix.cpp
Normal file
247
test/gpu_device_matrix.cpp
Normal file
@@ -0,0 +1,247 @@
|
||||
// This file is part of Eigen, a lightweight C++ template library
|
||||
// for linear algebra.
|
||||
//
|
||||
// Copyright (C) 2026 Rasmus Munk Larsen <rmlarsen@gmail.com>
|
||||
//
|
||||
// This Source Code Form is subject to the terms of the Mozilla
|
||||
// Public License v. 2.0. If a copy of the MPL was not distributed
|
||||
// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
|
||||
// Tests for DeviceMatrix and HostTransfer: typed RAII GPU memory wrapper.
|
||||
// No cuSOLVER dependency — only CUDA runtime.
|
||||
|
||||
#define EIGEN_USE_GPU
|
||||
#include "main.h"
|
||||
#include <Eigen/GPU>
|
||||
|
||||
using namespace Eigen;
|
||||
|
||||
// ---- Default construction ---------------------------------------------------
|
||||
|
||||
void test_default_construct() {
|
||||
DeviceMatrix<double> dm;
|
||||
VERIFY(dm.empty());
|
||||
VERIFY_IS_EQUAL(dm.rows(), 0);
|
||||
VERIFY_IS_EQUAL(dm.cols(), 0);
|
||||
VERIFY(dm.data() == nullptr);
|
||||
VERIFY_IS_EQUAL(dm.sizeInBytes(), size_t(0));
|
||||
}
|
||||
|
||||
// ---- Allocate uninitialized -------------------------------------------------
|
||||
|
||||
template <typename Scalar>
|
||||
void test_allocate(Index rows, Index cols) {
|
||||
DeviceMatrix<Scalar> dm(rows, cols);
|
||||
VERIFY(!dm.empty());
|
||||
VERIFY_IS_EQUAL(dm.rows(), rows);
|
||||
VERIFY_IS_EQUAL(dm.cols(), cols);
|
||||
VERIFY_IS_EQUAL(dm.outerStride(), rows);
|
||||
VERIFY(dm.data() != nullptr);
|
||||
VERIFY_IS_EQUAL(dm.sizeInBytes(), size_t(rows) * size_t(cols) * sizeof(Scalar));
|
||||
}
|
||||
|
||||
// ---- fromHost / toHost roundtrip (synchronous) ------------------------------
|
||||
|
||||
template <typename Scalar>
|
||||
void test_roundtrip(Index rows, Index cols) {
|
||||
using MatrixType = Matrix<Scalar, Dynamic, Dynamic>;
|
||||
MatrixType host = MatrixType::Random(rows, cols);
|
||||
|
||||
auto dm = DeviceMatrix<Scalar>::fromHost(host);
|
||||
VERIFY_IS_EQUAL(dm.rows(), rows);
|
||||
VERIFY_IS_EQUAL(dm.cols(), cols);
|
||||
VERIFY(!dm.empty());
|
||||
|
||||
MatrixType result = dm.toHost();
|
||||
VERIFY_IS_EQUAL(result.rows(), rows);
|
||||
VERIFY_IS_EQUAL(result.cols(), cols);
|
||||
VERIFY_IS_APPROX(result, host);
|
||||
}
|
||||
|
||||
// ---- fromHostAsync / toHostAsync roundtrip -----------------------------------
|
||||
|
||||
template <typename Scalar>
|
||||
void test_roundtrip_async(Index rows, Index cols) {
|
||||
using MatrixType = Matrix<Scalar, Dynamic, Dynamic>;
|
||||
MatrixType host = MatrixType::Random(rows, cols);
|
||||
|
||||
cudaStream_t stream;
|
||||
EIGEN_CUDA_RUNTIME_CHECK(cudaStreamCreate(&stream));
|
||||
|
||||
// Async upload from raw pointer.
|
||||
auto dm = DeviceMatrix<Scalar>::fromHostAsync(host.data(), rows, cols, rows, stream);
|
||||
VERIFY_IS_EQUAL(dm.rows(), rows);
|
||||
VERIFY_IS_EQUAL(dm.cols(), cols);
|
||||
|
||||
// Async download via HostTransfer future.
|
||||
auto transfer = dm.toHostAsync(stream);
|
||||
|
||||
// get() blocks and returns the matrix.
|
||||
MatrixType result = transfer.get();
|
||||
VERIFY_IS_APPROX(result, host);
|
||||
|
||||
EIGEN_CUDA_RUNTIME_CHECK(cudaStreamDestroy(stream));
|
||||
}
|
||||
|
||||
// ---- HostTransfer::ready() and idempotent get() -----------------------------
|
||||
|
||||
void test_host_transfer_ready() {
|
||||
using MatrixType = Matrix<double, Dynamic, Dynamic>;
|
||||
MatrixType host = MatrixType::Random(100, 100);
|
||||
|
||||
auto dm = DeviceMatrix<double>::fromHost(host);
|
||||
auto transfer = dm.toHostAsync();
|
||||
|
||||
// After get(), ready() must return true.
|
||||
MatrixType result = transfer.get();
|
||||
VERIFY(transfer.ready());
|
||||
VERIFY_IS_APPROX(result, host);
|
||||
|
||||
// get() is idempotent.
|
||||
MatrixType& result2 = transfer.get();
|
||||
VERIFY_IS_APPROX(result2, host);
|
||||
}
|
||||
|
||||
// ---- HostTransfer move ------------------------------------------------------
|
||||
|
||||
void test_host_transfer_move() {
|
||||
using MatrixType = Matrix<double, Dynamic, Dynamic>;
|
||||
MatrixType host = MatrixType::Random(50, 50);
|
||||
|
||||
auto dm = DeviceMatrix<double>::fromHost(host);
|
||||
auto transfer = dm.toHostAsync();
|
||||
|
||||
HostTransfer<double> moved(std::move(transfer));
|
||||
MatrixType result = moved.get();
|
||||
VERIFY_IS_APPROX(result, host);
|
||||
}
|
||||
|
||||
// ---- clone() produces independent copy --------------------------------------
|
||||
|
||||
template <typename Scalar>
|
||||
void test_clone(Index rows, Index cols) {
|
||||
using MatrixType = Matrix<Scalar, Dynamic, Dynamic>;
|
||||
MatrixType host = MatrixType::Random(rows, cols);
|
||||
|
||||
auto dm = DeviceMatrix<Scalar>::fromHost(host);
|
||||
auto cloned = dm.clone();
|
||||
|
||||
// Overwrite original with different data.
|
||||
MatrixType other = MatrixType::Random(rows, cols);
|
||||
dm = DeviceMatrix<Scalar>::fromHost(other);
|
||||
|
||||
// Clone still holds the original data.
|
||||
MatrixType clone_result = cloned.toHost();
|
||||
VERIFY_IS_APPROX(clone_result, host);
|
||||
|
||||
// Original holds the new data.
|
||||
MatrixType dm_result = dm.toHost();
|
||||
VERIFY_IS_APPROX(dm_result, other);
|
||||
}
|
||||
|
||||
// ---- Move construct ---------------------------------------------------------
|
||||
|
||||
template <typename Scalar>
|
||||
void test_move_construct(Index rows, Index cols) {
|
||||
using MatrixType = Matrix<Scalar, Dynamic, Dynamic>;
|
||||
MatrixType host = MatrixType::Random(rows, cols);
|
||||
|
||||
auto dm = DeviceMatrix<Scalar>::fromHost(host);
|
||||
DeviceMatrix<Scalar> moved(std::move(dm));
|
||||
|
||||
VERIFY(dm.empty());
|
||||
VERIFY(dm.data() == nullptr);
|
||||
|
||||
VERIFY_IS_EQUAL(moved.rows(), rows);
|
||||
VERIFY_IS_EQUAL(moved.cols(), cols);
|
||||
MatrixType result = moved.toHost();
|
||||
VERIFY_IS_APPROX(result, host);
|
||||
}
|
||||
|
||||
// ---- Move assign ------------------------------------------------------------
|
||||
|
||||
template <typename Scalar>
|
||||
void test_move_assign(Index rows, Index cols) {
|
||||
using MatrixType = Matrix<Scalar, Dynamic, Dynamic>;
|
||||
MatrixType host = MatrixType::Random(rows, cols);
|
||||
|
||||
auto dm = DeviceMatrix<Scalar>::fromHost(host);
|
||||
DeviceMatrix<Scalar> dest;
|
||||
dest = std::move(dm);
|
||||
|
||||
VERIFY(dm.empty());
|
||||
VERIFY_IS_EQUAL(dest.rows(), rows);
|
||||
MatrixType result = dest.toHost();
|
||||
VERIFY_IS_APPROX(result, host);
|
||||
}
|
||||
|
||||
// ---- resize() ---------------------------------------------------------------
|
||||
|
||||
void test_resize() {
|
||||
DeviceMatrix<double> dm(10, 20);
|
||||
VERIFY_IS_EQUAL(dm.rows(), 10);
|
||||
VERIFY_IS_EQUAL(dm.cols(), 20);
|
||||
|
||||
dm.resize(50, 30);
|
||||
VERIFY_IS_EQUAL(dm.rows(), 50);
|
||||
VERIFY_IS_EQUAL(dm.cols(), 30);
|
||||
VERIFY_IS_EQUAL(dm.outerStride(), 50);
|
||||
VERIFY(dm.data() != nullptr);
|
||||
|
||||
// Resize to same dimensions is a no-op.
|
||||
double* ptr_before = dm.data();
|
||||
dm.resize(50, 30);
|
||||
VERIFY(dm.data() == ptr_before);
|
||||
}
|
||||
|
||||
// ---- Empty / 0x0 matrix -----------------------------------------------------
|
||||
|
||||
void test_empty() {
|
||||
using MatrixType = Matrix<double, Dynamic, Dynamic>;
|
||||
MatrixType empty_mat(0, 0);
|
||||
|
||||
auto dm = DeviceMatrix<double>::fromHost(empty_mat);
|
||||
VERIFY(dm.empty());
|
||||
VERIFY_IS_EQUAL(dm.rows(), 0);
|
||||
VERIFY_IS_EQUAL(dm.cols(), 0);
|
||||
|
||||
MatrixType result = dm.toHost();
|
||||
VERIFY_IS_EQUAL(result.rows(), 0);
|
||||
VERIFY_IS_EQUAL(result.cols(), 0);
|
||||
}
|
||||
|
||||
// ---- Per-scalar driver ------------------------------------------------------
|
||||
|
||||
template <typename Scalar>
|
||||
void test_scalar() {
|
||||
// Square.
|
||||
CALL_SUBTEST(test_roundtrip<Scalar>(1, 1));
|
||||
CALL_SUBTEST(test_roundtrip<Scalar>(64, 64));
|
||||
CALL_SUBTEST(test_roundtrip<Scalar>(256, 256));
|
||||
|
||||
// Rectangular.
|
||||
CALL_SUBTEST(test_roundtrip<Scalar>(100, 7));
|
||||
CALL_SUBTEST(test_roundtrip<Scalar>(7, 100));
|
||||
|
||||
// Async roundtrip.
|
||||
CALL_SUBTEST(test_roundtrip_async<Scalar>(64, 64));
|
||||
CALL_SUBTEST(test_roundtrip_async<Scalar>(100, 7));
|
||||
|
||||
CALL_SUBTEST(test_clone<Scalar>(64, 64));
|
||||
CALL_SUBTEST(test_move_construct<Scalar>(64, 64));
|
||||
CALL_SUBTEST(test_move_assign<Scalar>(64, 64));
|
||||
}
|
||||
|
||||
EIGEN_DECLARE_TEST(gpu_device_matrix) {
|
||||
CALL_SUBTEST(test_default_construct());
|
||||
CALL_SUBTEST(test_empty());
|
||||
CALL_SUBTEST(test_resize());
|
||||
CALL_SUBTEST(test_host_transfer_ready());
|
||||
CALL_SUBTEST(test_host_transfer_move());
|
||||
CALL_SUBTEST((test_allocate<float>(100, 50)));
|
||||
CALL_SUBTEST((test_allocate<double>(100, 50)));
|
||||
CALL_SUBTEST(test_scalar<float>());
|
||||
CALL_SUBTEST(test_scalar<double>());
|
||||
CALL_SUBTEST(test_scalar<std::complex<float>>());
|
||||
CALL_SUBTEST(test_scalar<std::complex<double>>());
|
||||
}
|
||||
110
test/gpu_library_example.cu
Normal file
110
test/gpu_library_example.cu
Normal file
@@ -0,0 +1,110 @@
|
||||
// This file is part of Eigen, a lightweight C++ template library
|
||||
// for linear algebra.
|
||||
//
|
||||
// Copyright (C) 2026 Rasmus Munk Larsen <rmlarsen@gmail.com>
|
||||
//
|
||||
// This Source Code Form is subject to the terms of the Mozilla
|
||||
// Public License v. 2.0. If a copy of the MPL was not distributed
|
||||
// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
|
||||
// Smoke test for GPU library test infrastructure.
|
||||
// Verifies GpuContext, GpuBuffer, and host<->device matrix transfers
|
||||
// without requiring any NVIDIA library (cuBLAS, cuSOLVER, etc.).
|
||||
|
||||
#define EIGEN_USE_GPU
|
||||
#include "main.h"
|
||||
#include "gpu_context.h"
|
||||
#include "gpu_library_test_helper.h"
|
||||
|
||||
using namespace Eigen;
|
||||
using namespace Eigen::test;
|
||||
|
||||
// Test that GpuContext initializes, reports valid device info, and owns a cuSOLVER handle.
|
||||
void test_gpu_context() {
|
||||
GpuContext ctx;
|
||||
VERIFY(ctx.device() >= 0);
|
||||
VERIFY(ctx.deviceProperties().major >= 7); // sm_70 minimum
|
||||
VERIFY(ctx.stream != nullptr);
|
||||
VERIFY(ctx.cusolver != nullptr);
|
||||
std::cout << " GPU: " << ctx.deviceProperties().name << " (sm_" << ctx.deviceProperties().major
|
||||
<< ctx.deviceProperties().minor << ")\n";
|
||||
}
|
||||
|
||||
// Test dense matrix roundtrip: host -> device -> host.
|
||||
template <typename MatrixType>
|
||||
void test_dense_roundtrip() {
|
||||
GpuContext ctx;
|
||||
const Index rows = 64;
|
||||
const Index cols = 32;
|
||||
|
||||
MatrixType A = MatrixType::Random(rows, cols);
|
||||
auto buf = gpu_copy_to_device(ctx.stream, A);
|
||||
VERIFY(buf.data != nullptr);
|
||||
VERIFY(buf.size == rows * cols);
|
||||
|
||||
MatrixType B(rows, cols);
|
||||
B.setZero();
|
||||
gpu_copy_to_host(ctx.stream, buf, B);
|
||||
ctx.synchronize();
|
||||
|
||||
VERIFY_IS_EQUAL(A, B);
|
||||
}
|
||||
|
||||
// Test GpuBuffer RAII: move semantics, async zero-init.
|
||||
void test_gpu_buffer() {
|
||||
GpuContext ctx;
|
||||
|
||||
GpuBuffer<float> a(128);
|
||||
VERIFY(a.data != nullptr);
|
||||
VERIFY(a.size == 128);
|
||||
|
||||
// Move construction.
|
||||
GpuBuffer<float> b(std::move(a));
|
||||
VERIFY(a.data == nullptr);
|
||||
VERIFY(b.data != nullptr);
|
||||
VERIFY(b.size == 128);
|
||||
|
||||
// Move assignment.
|
||||
GpuBuffer<float> c;
|
||||
c = std::move(b);
|
||||
VERIFY(b.data == nullptr);
|
||||
VERIFY(c.data != nullptr);
|
||||
|
||||
// setZeroAsync.
|
||||
c.setZeroAsync(ctx.stream);
|
||||
ctx.synchronize();
|
||||
|
||||
std::vector<float> host(128);
|
||||
GPU_CHECK(cudaMemcpy(host.data(), c.data, 128 * sizeof(float), cudaMemcpyDeviceToHost));
|
||||
for (int i = 0; i < 128; ++i) {
|
||||
VERIFY_IS_EQUAL(host[i], 0.0f);
|
||||
}
|
||||
}
|
||||
|
||||
// Test with vectors (1D).
|
||||
template <typename Scalar>
|
||||
void test_vector_roundtrip() {
|
||||
GpuContext ctx;
|
||||
const Index n = 256;
|
||||
Matrix<Scalar, Dynamic, 1> v = Matrix<Scalar, Dynamic, 1>::Random(n);
|
||||
auto buf = gpu_copy_to_device(ctx.stream, v);
|
||||
|
||||
Matrix<Scalar, Dynamic, 1> w(n);
|
||||
w.setZero();
|
||||
gpu_copy_to_host(ctx.stream, buf, w);
|
||||
ctx.synchronize();
|
||||
|
||||
VERIFY_IS_EQUAL(v, w);
|
||||
}
|
||||
|
||||
EIGEN_DECLARE_TEST(gpu_library_example) {
|
||||
CALL_SUBTEST(test_gpu_context());
|
||||
CALL_SUBTEST(test_gpu_buffer());
|
||||
CALL_SUBTEST(test_dense_roundtrip<MatrixXf>());
|
||||
CALL_SUBTEST(test_dense_roundtrip<MatrixXd>());
|
||||
CALL_SUBTEST((test_dense_roundtrip<Matrix<float, Dynamic, Dynamic, RowMajor>>()));
|
||||
CALL_SUBTEST((test_dense_roundtrip<Matrix<double, Dynamic, Dynamic, RowMajor>>()));
|
||||
CALL_SUBTEST(test_vector_roundtrip<float>());
|
||||
CALL_SUBTEST(test_vector_roundtrip<double>());
|
||||
CALL_SUBTEST(test_vector_roundtrip<std::complex<float>>());
|
||||
}
|
||||
90
test/gpu_library_test_helper.h
Normal file
90
test/gpu_library_test_helper.h
Normal file
@@ -0,0 +1,90 @@
|
||||
// This file is part of Eigen, a lightweight C++ template library
|
||||
// for linear algebra.
|
||||
//
|
||||
// Copyright (C) 2026 Rasmus Munk Larsen <rmlarsen@gmail.com>
|
||||
//
|
||||
// This Source Code Form is subject to the terms of the Mozilla
|
||||
// Public License v. 2.0. If a copy of the MPL was not distributed
|
||||
// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
|
||||
#ifndef EIGEN_TEST_GPU_LIBRARY_TEST_HELPER_H
|
||||
#define EIGEN_TEST_GPU_LIBRARY_TEST_HELPER_H
|
||||
|
||||
// Helpers for GPU tests that call NVIDIA library APIs (cuBLAS, cuSOLVER, etc.)
|
||||
// from the host side. Provides RAII GPU memory management and async matrix transfer.
|
||||
//
|
||||
// This is separate from gpu_common.h (element-parallel device kernels) and
|
||||
// gpu_test_helper.h (serialization-based device kernels). Those patterns run
|
||||
// user functors inside GPU kernels. This helper is for host-orchestrated tests
|
||||
// that call library APIs which launch their own kernels internally.
|
||||
//
|
||||
// All transfers use an explicit stream and cudaMemcpyAsync. Callers must
|
||||
// synchronize (ctx.synchronize() or cudaStreamSynchronize) before reading
|
||||
// results back on the host.
|
||||
|
||||
#include "gpu_test_helper.h"
|
||||
|
||||
namespace Eigen {
|
||||
namespace test {
|
||||
|
||||
// RAII wrapper for GPU device memory. Prevents leaks when VERIFY macros abort.
|
||||
template <typename Scalar>
|
||||
struct GpuBuffer {
|
||||
Scalar* data = nullptr;
|
||||
Index size = 0;
|
||||
|
||||
GpuBuffer() = default;
|
||||
|
||||
explicit GpuBuffer(Index n) : size(n) { GPU_CHECK(gpuMalloc(reinterpret_cast<void**>(&data), n * sizeof(Scalar))); }
|
||||
|
||||
~GpuBuffer() {
|
||||
if (data) GPU_CHECK(gpuFree(data));
|
||||
}
|
||||
|
||||
// Move-only.
|
||||
GpuBuffer(GpuBuffer&& other) noexcept : data(other.data), size(other.size) {
|
||||
other.data = nullptr;
|
||||
other.size = 0;
|
||||
}
|
||||
GpuBuffer& operator=(GpuBuffer&& other) noexcept {
|
||||
if (this != &other) {
|
||||
if (data) GPU_CHECK(gpuFree(data));
|
||||
data = other.data;
|
||||
size = other.size;
|
||||
other.data = nullptr;
|
||||
other.size = 0;
|
||||
}
|
||||
return *this;
|
||||
}
|
||||
|
||||
GpuBuffer(const GpuBuffer&) = delete;
|
||||
GpuBuffer& operator=(const GpuBuffer&) = delete;
|
||||
|
||||
// Async zero the buffer on the given stream.
|
||||
void setZeroAsync(cudaStream_t stream) { GPU_CHECK(cudaMemsetAsync(data, 0, size * sizeof(Scalar), stream)); }
|
||||
};
|
||||
|
||||
// Copy a dense Eigen matrix to a new GPU buffer, async on the given stream.
|
||||
// Caller must synchronize before the host matrix is freed or modified.
|
||||
template <typename Derived>
|
||||
GpuBuffer<typename Derived::Scalar> gpu_copy_to_device(cudaStream_t stream, const MatrixBase<Derived>& host_mat) {
|
||||
using Scalar = typename Derived::Scalar;
|
||||
const auto& mat = host_mat.derived();
|
||||
GpuBuffer<Scalar> buf(mat.size());
|
||||
GPU_CHECK(cudaMemcpyAsync(buf.data, mat.data(), mat.size() * sizeof(Scalar), cudaMemcpyHostToDevice, stream));
|
||||
return buf;
|
||||
}
|
||||
|
||||
// Copy GPU buffer contents back to a dense Eigen matrix, async on the given stream.
|
||||
// Caller must synchronize before reading from host_mat.
|
||||
template <typename Scalar, typename Derived>
|
||||
void gpu_copy_to_host(cudaStream_t stream, const GpuBuffer<Scalar>& buf, MatrixBase<Derived>& host_mat) {
|
||||
auto& mat = host_mat.derived();
|
||||
eigen_assert(buf.size == mat.size());
|
||||
GPU_CHECK(cudaMemcpyAsync(mat.data(), buf.data, mat.size() * sizeof(Scalar), cudaMemcpyDeviceToHost, stream));
|
||||
}
|
||||
|
||||
} // namespace test
|
||||
} // namespace Eigen
|
||||
|
||||
#endif // EIGEN_TEST_GPU_LIBRARY_TEST_HELPER_H
|
||||
Reference in New Issue
Block a user