mirror of
https://gitlab.com/libeigen/eigen.git
synced 2026-04-10 11:34:33 +08:00
GPU: Add sparse solvers, FFT, and SpMV (cuDSS, cuFFT, cuSPARSE)
Add GPU sparse direct solvers (Cholesky, LDL^T, LU) via cuDSS, 1D/2D FFT via cuFFT with plan caching, and sparse matrix-vector/matrix multiply (SpMV/SpMM) via cuSPARSE. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
Rasmus Munk Larsen
11
Eigen/GPU
11
Eigen/GPU
@@ -50,6 +50,17 @@
|
||||
#include "src/GPU/GpuQR.h"
|
||||
#include "src/GPU/GpuSVD.h"
|
||||
#include "src/GPU/GpuEigenSolver.h"
|
||||
#include "src/GPU/CuFftSupport.h"
|
||||
#include "src/GPU/GpuFFT.h"
|
||||
#include "src/GPU/CuSparseSupport.h"
|
||||
#include "src/GPU/GpuSparseContext.h"
|
||||
#ifdef EIGEN_CUDSS
|
||||
#include "src/GPU/CuDssSupport.h"
|
||||
#include "src/GPU/GpuSparseSolverBase.h"
|
||||
#include "src/GPU/GpuSparseLLT.h"
|
||||
#include "src/GPU/GpuSparseLDLT.h"
|
||||
#include "src/GPU/GpuSparseLU.h"
|
||||
#endif
|
||||
// IWYU pragma: end_exports
|
||||
#endif
|
||||
|
||||
|
||||
134
Eigen/src/GPU/CuDssSupport.h
Normal file
134
Eigen/src/GPU/CuDssSupport.h
Normal file
@@ -0,0 +1,134 @@
|
||||
// This file is part of Eigen, a lightweight C++ template library
|
||||
// for linear algebra.
|
||||
//
|
||||
// Copyright (C) 2026 Rasmus Munk Larsen <rmlarsen@gmail.com>
|
||||
//
|
||||
// This Source Code Form is subject to the terms of the Mozilla
|
||||
// Public License v. 2.0. If a copy of the MPL was not distributed
|
||||
// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
|
||||
// cuDSS support utilities: error checking macro, type mapping.
|
||||
//
|
||||
// cuDSS is NVIDIA's sparse direct solver library, supporting Cholesky (LL^T),
|
||||
// LDL^T, and LU factorization on GPU. It requires CUDA 12.0+ and is
|
||||
// distributed separately from the CUDA Toolkit.
|
||||
|
||||
#ifndef EIGEN_GPU_CUDSS_SUPPORT_H
|
||||
#define EIGEN_GPU_CUDSS_SUPPORT_H
|
||||
|
||||
// IWYU pragma: private
|
||||
#include "./InternalHeaderCheck.h"
|
||||
|
||||
#include "./GpuSupport.h"
|
||||
#include <cudss.h>
|
||||
|
||||
namespace Eigen {
|
||||
namespace internal {
|
||||
|
||||
// ---- Error checking ---------------------------------------------------------
|
||||
|
||||
#define EIGEN_CUDSS_CHECK(x) \
|
||||
do { \
|
||||
cudssStatus_t _s = (x); \
|
||||
eigen_assert(_s == CUDSS_STATUS_SUCCESS && "cuDSS call failed: " #x); \
|
||||
EIGEN_UNUSED_VARIABLE(_s); \
|
||||
} while (0)
|
||||
|
||||
// ---- Scalar → cudssMatrixType_t for SPD/HPD ---------------------------------
|
||||
|
||||
template <typename Scalar>
|
||||
struct cudss_spd_type;
|
||||
|
||||
template <>
|
||||
struct cudss_spd_type<float> {
|
||||
static constexpr cudssMatrixType_t value = CUDSS_MTYPE_SPD;
|
||||
};
|
||||
template <>
|
||||
struct cudss_spd_type<double> {
|
||||
static constexpr cudssMatrixType_t value = CUDSS_MTYPE_SPD;
|
||||
};
|
||||
template <>
|
||||
struct cudss_spd_type<std::complex<float>> {
|
||||
static constexpr cudssMatrixType_t value = CUDSS_MTYPE_HPD;
|
||||
};
|
||||
template <>
|
||||
struct cudss_spd_type<std::complex<double>> {
|
||||
static constexpr cudssMatrixType_t value = CUDSS_MTYPE_HPD;
|
||||
};
|
||||
|
||||
// ---- Scalar → cudssMatrixType_t for symmetric/Hermitian ---------------------
|
||||
|
||||
template <typename Scalar>
|
||||
struct cudss_symmetric_type;
|
||||
|
||||
template <>
|
||||
struct cudss_symmetric_type<float> {
|
||||
static constexpr cudssMatrixType_t value = CUDSS_MTYPE_SYMMETRIC;
|
||||
};
|
||||
template <>
|
||||
struct cudss_symmetric_type<double> {
|
||||
static constexpr cudssMatrixType_t value = CUDSS_MTYPE_SYMMETRIC;
|
||||
};
|
||||
template <>
|
||||
struct cudss_symmetric_type<std::complex<float>> {
|
||||
static constexpr cudssMatrixType_t value = CUDSS_MTYPE_HERMITIAN;
|
||||
};
|
||||
template <>
|
||||
struct cudss_symmetric_type<std::complex<double>> {
|
||||
static constexpr cudssMatrixType_t value = CUDSS_MTYPE_HERMITIAN;
|
||||
};
|
||||
|
||||
// ---- StorageIndex → cudaDataType_t ------------------------------------------
|
||||
|
||||
template <typename StorageIndex>
|
||||
struct cudss_index_type;
|
||||
|
||||
template <>
|
||||
struct cudss_index_type<int> {
|
||||
static constexpr cudaDataType_t value = CUDA_R_32I;
|
||||
};
|
||||
template <>
|
||||
struct cudss_index_type<int64_t> {
|
||||
static constexpr cudaDataType_t value = CUDA_R_64I;
|
||||
};
|
||||
|
||||
// ---- UpLo → cudssMatrixViewType_t -------------------------------------------
|
||||
// For symmetric matrices stored as CSC (ColMajor), cuDSS sees CSR of A^T.
|
||||
// Since A = A^T, the data is the same, but the triangle view must be swapped.
|
||||
|
||||
template <int UpLo, int StorageOrder>
|
||||
struct cudss_view_type;
|
||||
|
||||
// ColMajor (CSC) passed as CSR: lower ↔ upper swap.
|
||||
template <>
|
||||
struct cudss_view_type<Lower, ColMajor> {
|
||||
static constexpr cudssMatrixViewType_t value = CUDSS_MVIEW_UPPER;
|
||||
};
|
||||
template <>
|
||||
struct cudss_view_type<Upper, ColMajor> {
|
||||
static constexpr cudssMatrixViewType_t value = CUDSS_MVIEW_LOWER;
|
||||
};
|
||||
|
||||
// RowMajor (CSR) passed directly: no swap needed.
|
||||
template <>
|
||||
struct cudss_view_type<Lower, RowMajor> {
|
||||
static constexpr cudssMatrixViewType_t value = CUDSS_MVIEW_LOWER;
|
||||
};
|
||||
template <>
|
||||
struct cudss_view_type<Upper, RowMajor> {
|
||||
static constexpr cudssMatrixViewType_t value = CUDSS_MVIEW_UPPER;
|
||||
};
|
||||
|
||||
} // namespace internal
|
||||
|
||||
// ---- Ordering enum ----------------------------------------------------------
|
||||
|
||||
enum class GpuSparseOrdering {
|
||||
AMD, // Default fill-reducing ordering
|
||||
METIS, // METIS nested dissection
|
||||
RCM // Reverse Cuthill-McKee
|
||||
};
|
||||
|
||||
} // namespace Eigen
|
||||
|
||||
#endif // EIGEN_GPU_CUDSS_SUPPORT_H
|
||||
103
Eigen/src/GPU/CuFftSupport.h
Normal file
103
Eigen/src/GPU/CuFftSupport.h
Normal file
@@ -0,0 +1,103 @@
|
||||
// This file is part of Eigen, a lightweight C++ template library
|
||||
// for linear algebra.
|
||||
//
|
||||
// Copyright (C) 2026 Rasmus Munk Larsen <rmlarsen@gmail.com>
|
||||
//
|
||||
// This Source Code Form is subject to the terms of the Mozilla
|
||||
// Public License v. 2.0. If a copy of the MPL was not distributed
|
||||
// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
|
||||
// cuFFT support utilities: error checking macro, type mapping.
|
||||
|
||||
#ifndef EIGEN_GPU_CUFFT_SUPPORT_H
|
||||
#define EIGEN_GPU_CUFFT_SUPPORT_H
|
||||
|
||||
// IWYU pragma: private
|
||||
#include "./InternalHeaderCheck.h"
|
||||
|
||||
#include "./GpuSupport.h"
|
||||
#include <cufft.h>
|
||||
|
||||
namespace Eigen {
|
||||
namespace internal {
|
||||
|
||||
// ---- Error checking ---------------------------------------------------------
|
||||
|
||||
#define EIGEN_CUFFT_CHECK(x) \
|
||||
do { \
|
||||
cufftResult _r = (x); \
|
||||
eigen_assert(_r == CUFFT_SUCCESS && "cuFFT call failed: " #x); \
|
||||
EIGEN_UNUSED_VARIABLE(_r); \
|
||||
} while (0)
|
||||
|
||||
// ---- Scalar → cufftType traits ----------------------------------------------
|
||||
|
||||
template <typename Scalar>
|
||||
struct cufft_c2c_type;
|
||||
|
||||
template <>
|
||||
struct cufft_c2c_type<float> {
|
||||
static constexpr cufftType value = CUFFT_C2C;
|
||||
};
|
||||
template <>
|
||||
struct cufft_c2c_type<double> {
|
||||
static constexpr cufftType value = CUFFT_Z2Z;
|
||||
};
|
||||
|
||||
template <typename Scalar>
|
||||
struct cufft_r2c_type;
|
||||
|
||||
template <>
|
||||
struct cufft_r2c_type<float> {
|
||||
static constexpr cufftType value = CUFFT_R2C;
|
||||
};
|
||||
template <>
|
||||
struct cufft_r2c_type<double> {
|
||||
static constexpr cufftType value = CUFFT_D2Z;
|
||||
};
|
||||
|
||||
template <typename Scalar>
|
||||
struct cufft_c2r_type;
|
||||
|
||||
template <>
|
||||
struct cufft_c2r_type<float> {
|
||||
static constexpr cufftType value = CUFFT_C2R;
|
||||
};
|
||||
template <>
|
||||
struct cufft_c2r_type<double> {
|
||||
static constexpr cufftType value = CUFFT_Z2D;
|
||||
};
|
||||
|
||||
// ---- Type-dispatched cuFFT execution ----------------------------------------
|
||||
|
||||
// C2C
|
||||
inline cufftResult cufftExecC2C_dispatch(cufftHandle plan, std::complex<float>* in, std::complex<float>* out,
|
||||
int direction) {
|
||||
return cufftExecC2C(plan, reinterpret_cast<cufftComplex*>(in), reinterpret_cast<cufftComplex*>(out), direction);
|
||||
}
|
||||
inline cufftResult cufftExecC2C_dispatch(cufftHandle plan, std::complex<double>* in, std::complex<double>* out,
|
||||
int direction) {
|
||||
return cufftExecZ2Z(plan, reinterpret_cast<cufftDoubleComplex*>(in), reinterpret_cast<cufftDoubleComplex*>(out),
|
||||
direction);
|
||||
}
|
||||
|
||||
// R2C
|
||||
inline cufftResult cufftExecR2C_dispatch(cufftHandle plan, float* in, std::complex<float>* out) {
|
||||
return cufftExecR2C(plan, in, reinterpret_cast<cufftComplex*>(out));
|
||||
}
|
||||
inline cufftResult cufftExecR2C_dispatch(cufftHandle plan, double* in, std::complex<double>* out) {
|
||||
return cufftExecD2Z(plan, in, reinterpret_cast<cufftDoubleComplex*>(out));
|
||||
}
|
||||
|
||||
// C2R
|
||||
inline cufftResult cufftExecC2R_dispatch(cufftHandle plan, std::complex<float>* in, float* out) {
|
||||
return cufftExecC2R(plan, reinterpret_cast<cufftComplex*>(in), out);
|
||||
}
|
||||
inline cufftResult cufftExecC2R_dispatch(cufftHandle plan, std::complex<double>* in, double* out) {
|
||||
return cufftExecZ2D(plan, reinterpret_cast<cufftDoubleComplex*>(in), out);
|
||||
}
|
||||
|
||||
} // namespace internal
|
||||
} // namespace Eigen
|
||||
|
||||
#endif // EIGEN_GPU_CUFFT_SUPPORT_H
|
||||
34
Eigen/src/GPU/CuSparseSupport.h
Normal file
34
Eigen/src/GPU/CuSparseSupport.h
Normal file
@@ -0,0 +1,34 @@
|
||||
// This file is part of Eigen, a lightweight C++ template library
|
||||
// for linear algebra.
|
||||
//
|
||||
// Copyright (C) 2026 Rasmus Munk Larsen <rmlarsen@gmail.com>
|
||||
//
|
||||
// This Source Code Form is subject to the terms of the Mozilla
|
||||
// Public License v. 2.0. If a copy of the MPL was not distributed
|
||||
// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
|
||||
// cuSPARSE support utilities: error checking macro.
|
||||
|
||||
#ifndef EIGEN_GPU_CUSPARSE_SUPPORT_H
|
||||
#define EIGEN_GPU_CUSPARSE_SUPPORT_H
|
||||
|
||||
// IWYU pragma: private
|
||||
#include "./InternalHeaderCheck.h"
|
||||
|
||||
#include "./GpuSupport.h"
|
||||
#include <cusparse.h>
|
||||
|
||||
namespace Eigen {
|
||||
namespace internal {
|
||||
|
||||
#define EIGEN_CUSPARSE_CHECK(x) \
|
||||
do { \
|
||||
cusparseStatus_t _s = (x); \
|
||||
eigen_assert(_s == CUSPARSE_STATUS_SUCCESS && "cuSPARSE call failed: " #x); \
|
||||
EIGEN_UNUSED_VARIABLE(_s); \
|
||||
} while (0)
|
||||
|
||||
} // namespace internal
|
||||
} // namespace Eigen
|
||||
|
||||
#endif // EIGEN_GPU_CUSPARSE_SUPPORT_H
|
||||
@@ -116,6 +116,10 @@ class GpuSelfAdjointEigenSolver {
|
||||
Index cols() const { return n_; }
|
||||
Index rows() const { return n_; }
|
||||
|
||||
// TODO: Add device-side accessors (deviceEigenvalues(), deviceEigenvectors())
|
||||
// returning DeviceMatrix views of the internal buffers, so users can chain
|
||||
// GPU operations without round-tripping through host memory.
|
||||
|
||||
/** Eigenvalues in ascending order. Downloads from device. */
|
||||
RealVector eigenvalues() const {
|
||||
sync_info();
|
||||
|
||||
308
Eigen/src/GPU/GpuFFT.h
Normal file
308
Eigen/src/GPU/GpuFFT.h
Normal file
@@ -0,0 +1,308 @@
|
||||
// This file is part of Eigen, a lightweight C++ template library
|
||||
// for linear algebra.
|
||||
//
|
||||
// Copyright (C) 2026 Rasmus Munk Larsen <rmlarsen@gmail.com>
|
||||
//
|
||||
// This Source Code Form is subject to the terms of the Mozilla
|
||||
// Public License v. 2.0. If a copy of the MPL was not distributed
|
||||
// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
|
||||
// GPU FFT via cuFFT.
|
||||
//
|
||||
// Standalone GPU FFT class with plan caching. Supports 1D and 2D transforms:
|
||||
// C2C (complex-to-complex), R2C (real-to-complex), C2R (complex-to-real).
|
||||
//
|
||||
// Inverse transforms are scaled by 1/n (1D) or 1/(n*m) (2D) so that
|
||||
// inv(fwd(x)) == x, matching Eigen's FFT convention.
|
||||
//
|
||||
// cuFFT plans are cached by (size, type) and reused across calls.
|
||||
//
|
||||
// Usage:
|
||||
// GpuFFT<float> fft;
|
||||
// VectorXcf X = fft.fwd(x); // 1D C2C or R2C
|
||||
// VectorXcf y = fft.inv(X); // 1D C2C inverse
|
||||
// VectorXf r = fft.invReal(X, n); // 1D C2R inverse
|
||||
// MatrixXcf B = fft.fwd2d(A); // 2D C2C forward
|
||||
// MatrixXcf C = fft.inv2d(B); // 2D C2C inverse
|
||||
|
||||
#ifndef EIGEN_GPU_FFT_H
|
||||
#define EIGEN_GPU_FFT_H
|
||||
|
||||
// IWYU pragma: private
|
||||
#include "./InternalHeaderCheck.h"
|
||||
|
||||
#include "./CuFftSupport.h"
|
||||
#include "./CuBlasSupport.h"
|
||||
#include <map>
|
||||
|
||||
namespace Eigen {
|
||||
|
||||
template <typename Scalar_>
|
||||
class GpuFFT {
|
||||
public:
|
||||
using Scalar = Scalar_;
|
||||
using Complex = std::complex<Scalar>;
|
||||
using ComplexVector = Matrix<Complex, Dynamic, 1>;
|
||||
using RealVector = Matrix<Scalar, Dynamic, 1>;
|
||||
using ComplexMatrix = Matrix<Complex, Dynamic, Dynamic, ColMajor>;
|
||||
|
||||
GpuFFT() {
|
||||
EIGEN_CUDA_RUNTIME_CHECK(cudaStreamCreate(&stream_));
|
||||
EIGEN_CUBLAS_CHECK(cublasCreate(&cublas_));
|
||||
EIGEN_CUBLAS_CHECK(cublasSetStream(cublas_, stream_));
|
||||
}
|
||||
|
||||
~GpuFFT() {
|
||||
for (auto& kv : plans_) (void)cufftDestroy(kv.second);
|
||||
if (cublas_) (void)cublasDestroy(cublas_);
|
||||
if (stream_) (void)cudaStreamDestroy(stream_);
|
||||
}
|
||||
|
||||
GpuFFT(const GpuFFT&) = delete;
|
||||
GpuFFT& operator=(const GpuFFT&) = delete;
|
||||
|
||||
// ---- 1D Complex-to-Complex ------------------------------------------------
|
||||
|
||||
/** Forward 1D C2C FFT. */
|
||||
template <typename Derived>
|
||||
ComplexVector fwd(const MatrixBase<Derived>& x,
|
||||
typename std::enable_if<NumTraits<typename Derived::Scalar>::IsComplex>::type* = nullptr) {
|
||||
const ComplexVector input(x.derived());
|
||||
const int n = static_cast<int>(input.size());
|
||||
if (n == 0) return ComplexVector(0);
|
||||
|
||||
ensure_buffers(n * sizeof(Complex), n * sizeof(Complex));
|
||||
EIGEN_CUDA_RUNTIME_CHECK(
|
||||
cudaMemcpyAsync(d_in_.ptr, input.data(), n * sizeof(Complex), cudaMemcpyHostToDevice, stream_));
|
||||
|
||||
cufftHandle plan = get_plan_1d(n, internal::cufft_c2c_type<Scalar>::value);
|
||||
EIGEN_CUFFT_CHECK(internal::cufftExecC2C_dispatch(plan, static_cast<Complex*>(d_in_.ptr),
|
||||
static_cast<Complex*>(d_out_.ptr), CUFFT_FORWARD));
|
||||
|
||||
ComplexVector result(n);
|
||||
EIGEN_CUDA_RUNTIME_CHECK(
|
||||
cudaMemcpyAsync(result.data(), d_out_.ptr, n * sizeof(Complex), cudaMemcpyDeviceToHost, stream_));
|
||||
EIGEN_CUDA_RUNTIME_CHECK(cudaStreamSynchronize(stream_));
|
||||
return result;
|
||||
}
|
||||
|
||||
/** Inverse 1D C2C FFT. Scaled by 1/n. */
|
||||
template <typename Derived>
|
||||
ComplexVector inv(const MatrixBase<Derived>& X) {
|
||||
static_assert(NumTraits<typename Derived::Scalar>::IsComplex, "inv() requires complex input");
|
||||
const ComplexVector input(X.derived());
|
||||
const int n = static_cast<int>(input.size());
|
||||
if (n == 0) return ComplexVector(0);
|
||||
|
||||
ensure_buffers(n * sizeof(Complex), n * sizeof(Complex));
|
||||
EIGEN_CUDA_RUNTIME_CHECK(
|
||||
cudaMemcpyAsync(d_in_.ptr, input.data(), n * sizeof(Complex), cudaMemcpyHostToDevice, stream_));
|
||||
|
||||
cufftHandle plan = get_plan_1d(n, internal::cufft_c2c_type<Scalar>::value);
|
||||
EIGEN_CUFFT_CHECK(internal::cufftExecC2C_dispatch(plan, static_cast<Complex*>(d_in_.ptr),
|
||||
static_cast<Complex*>(d_out_.ptr), CUFFT_INVERSE));
|
||||
|
||||
// Scale by 1/n.
|
||||
scale_device(static_cast<Complex*>(d_out_.ptr), n, Scalar(1) / Scalar(n));
|
||||
|
||||
ComplexVector result(n);
|
||||
EIGEN_CUDA_RUNTIME_CHECK(
|
||||
cudaMemcpyAsync(result.data(), d_out_.ptr, n * sizeof(Complex), cudaMemcpyDeviceToHost, stream_));
|
||||
EIGEN_CUDA_RUNTIME_CHECK(cudaStreamSynchronize(stream_));
|
||||
return result;
|
||||
}
|
||||
|
||||
// ---- 1D Real-to-Complex ---------------------------------------------------
|
||||
|
||||
/** Forward 1D R2C FFT. Returns n/2+1 complex values (half-spectrum). */
|
||||
template <typename Derived>
|
||||
ComplexVector fwd(const MatrixBase<Derived>& x,
|
||||
typename std::enable_if<!NumTraits<typename Derived::Scalar>::IsComplex>::type* = nullptr) {
|
||||
const RealVector input(x.derived());
|
||||
const int n = static_cast<int>(input.size());
|
||||
if (n == 0) return ComplexVector(0);
|
||||
|
||||
const int n_complex = n / 2 + 1;
|
||||
ensure_buffers(n * sizeof(Scalar), n_complex * sizeof(Complex));
|
||||
EIGEN_CUDA_RUNTIME_CHECK(
|
||||
cudaMemcpyAsync(d_in_.ptr, input.data(), n * sizeof(Scalar), cudaMemcpyHostToDevice, stream_));
|
||||
|
||||
cufftHandle plan = get_plan_1d(n, internal::cufft_r2c_type<Scalar>::value);
|
||||
EIGEN_CUFFT_CHECK(
|
||||
internal::cufftExecR2C_dispatch(plan, static_cast<Scalar*>(d_in_.ptr), static_cast<Complex*>(d_out_.ptr)));
|
||||
|
||||
ComplexVector result(n_complex);
|
||||
EIGEN_CUDA_RUNTIME_CHECK(
|
||||
cudaMemcpyAsync(result.data(), d_out_.ptr, n_complex * sizeof(Complex), cudaMemcpyDeviceToHost, stream_));
|
||||
EIGEN_CUDA_RUNTIME_CHECK(cudaStreamSynchronize(stream_));
|
||||
return result;
|
||||
}
|
||||
|
||||
// ---- 1D Complex-to-Real ---------------------------------------------------
|
||||
|
||||
/** Inverse 1D C2R FFT. Input is n/2+1 complex values, output is nfft real values.
|
||||
* Scaled by 1/nfft. Caller must specify nfft (original real signal length). */
|
||||
template <typename Derived>
|
||||
RealVector invReal(const MatrixBase<Derived>& X, Index nfft) {
|
||||
static_assert(NumTraits<typename Derived::Scalar>::IsComplex, "invReal() requires complex input");
|
||||
const ComplexVector input(X.derived());
|
||||
const int n = static_cast<int>(nfft);
|
||||
const int n_complex = n / 2 + 1;
|
||||
eigen_assert(input.size() == n_complex);
|
||||
if (n == 0) return RealVector(0);
|
||||
|
||||
ensure_buffers(n_complex * sizeof(Complex), n * sizeof(Scalar));
|
||||
// cuFFT C2R may overwrite the input, so we copy to d_in_.
|
||||
EIGEN_CUDA_RUNTIME_CHECK(
|
||||
cudaMemcpyAsync(d_in_.ptr, input.data(), n_complex * sizeof(Complex), cudaMemcpyHostToDevice, stream_));
|
||||
|
||||
cufftHandle plan = get_plan_1d(n, internal::cufft_c2r_type<Scalar>::value);
|
||||
EIGEN_CUFFT_CHECK(
|
||||
internal::cufftExecC2R_dispatch(plan, static_cast<Complex*>(d_in_.ptr), static_cast<Scalar*>(d_out_.ptr)));
|
||||
|
||||
// Scale by 1/n.
|
||||
scale_device_real(static_cast<Scalar*>(d_out_.ptr), n, Scalar(1) / Scalar(n));
|
||||
|
||||
RealVector result(n);
|
||||
EIGEN_CUDA_RUNTIME_CHECK(
|
||||
cudaMemcpyAsync(result.data(), d_out_.ptr, n * sizeof(Scalar), cudaMemcpyDeviceToHost, stream_));
|
||||
EIGEN_CUDA_RUNTIME_CHECK(cudaStreamSynchronize(stream_));
|
||||
return result;
|
||||
}
|
||||
|
||||
// ---- 2D Complex-to-Complex ------------------------------------------------
|
||||
|
||||
/** Forward 2D C2C FFT. Input and output are rows x cols complex matrices. */
|
||||
template <typename Derived>
|
||||
ComplexMatrix fwd2d(const MatrixBase<Derived>& A) {
|
||||
static_assert(NumTraits<typename Derived::Scalar>::IsComplex, "fwd2d() requires complex input");
|
||||
const ComplexMatrix input(A.derived());
|
||||
const int rows = static_cast<int>(input.rows());
|
||||
const int cols = static_cast<int>(input.cols());
|
||||
if (rows == 0 || cols == 0) return ComplexMatrix(rows, cols);
|
||||
|
||||
const size_t total = static_cast<size_t>(rows) * static_cast<size_t>(cols) * sizeof(Complex);
|
||||
ensure_buffers(total, total);
|
||||
EIGEN_CUDA_RUNTIME_CHECK(cudaMemcpyAsync(d_in_.ptr, input.data(), total, cudaMemcpyHostToDevice, stream_));
|
||||
|
||||
cufftHandle plan = get_plan_2d(rows, cols, internal::cufft_c2c_type<Scalar>::value);
|
||||
EIGEN_CUFFT_CHECK(internal::cufftExecC2C_dispatch(plan, static_cast<Complex*>(d_in_.ptr),
|
||||
static_cast<Complex*>(d_out_.ptr), CUFFT_FORWARD));
|
||||
|
||||
ComplexMatrix result(rows, cols);
|
||||
EIGEN_CUDA_RUNTIME_CHECK(cudaMemcpyAsync(result.data(), d_out_.ptr, total, cudaMemcpyDeviceToHost, stream_));
|
||||
EIGEN_CUDA_RUNTIME_CHECK(cudaStreamSynchronize(stream_));
|
||||
return result;
|
||||
}
|
||||
|
||||
/** Inverse 2D C2C FFT. Scaled by 1/(rows*cols). */
|
||||
template <typename Derived>
|
||||
ComplexMatrix inv2d(const MatrixBase<Derived>& A) {
|
||||
static_assert(NumTraits<typename Derived::Scalar>::IsComplex, "inv2d() requires complex input");
|
||||
const ComplexMatrix input(A.derived());
|
||||
const int rows = static_cast<int>(input.rows());
|
||||
const int cols = static_cast<int>(input.cols());
|
||||
if (rows == 0 || cols == 0) return ComplexMatrix(rows, cols);
|
||||
|
||||
const size_t total = static_cast<size_t>(rows) * static_cast<size_t>(cols) * sizeof(Complex);
|
||||
ensure_buffers(total, total);
|
||||
EIGEN_CUDA_RUNTIME_CHECK(cudaMemcpyAsync(d_in_.ptr, input.data(), total, cudaMemcpyHostToDevice, stream_));
|
||||
|
||||
cufftHandle plan = get_plan_2d(rows, cols, internal::cufft_c2c_type<Scalar>::value);
|
||||
EIGEN_CUFFT_CHECK(internal::cufftExecC2C_dispatch(plan, static_cast<Complex*>(d_in_.ptr),
|
||||
static_cast<Complex*>(d_out_.ptr), CUFFT_INVERSE));
|
||||
|
||||
// Scale by 1/(rows*cols).
|
||||
const int total_elems = rows * cols;
|
||||
scale_device(static_cast<Complex*>(d_out_.ptr), total_elems, Scalar(1) / Scalar(total_elems));
|
||||
|
||||
ComplexMatrix result(rows, cols);
|
||||
EIGEN_CUDA_RUNTIME_CHECK(cudaMemcpyAsync(result.data(), d_out_.ptr, total, cudaMemcpyDeviceToHost, stream_));
|
||||
EIGEN_CUDA_RUNTIME_CHECK(cudaStreamSynchronize(stream_));
|
||||
return result;
|
||||
}
|
||||
|
||||
// ---- Accessors ------------------------------------------------------------
|
||||
|
||||
cudaStream_t stream() const { return stream_; }
|
||||
|
||||
private:
|
||||
cudaStream_t stream_ = nullptr;
|
||||
cublasHandle_t cublas_ = nullptr;
|
||||
std::map<int64_t, cufftHandle> plans_;
|
||||
internal::DeviceBuffer d_in_;
|
||||
internal::DeviceBuffer d_out_;
|
||||
size_t d_in_size_ = 0;
|
||||
size_t d_out_size_ = 0;
|
||||
|
||||
void ensure_buffers(size_t in_bytes, size_t out_bytes) {
|
||||
if (in_bytes > d_in_size_) {
|
||||
if (d_in_.ptr) EIGEN_CUDA_RUNTIME_CHECK(cudaStreamSynchronize(stream_));
|
||||
d_in_ = internal::DeviceBuffer(in_bytes);
|
||||
d_in_size_ = in_bytes;
|
||||
}
|
||||
if (out_bytes > d_out_size_) {
|
||||
if (d_out_.ptr) EIGEN_CUDA_RUNTIME_CHECK(cudaStreamSynchronize(stream_));
|
||||
d_out_ = internal::DeviceBuffer(out_bytes);
|
||||
d_out_size_ = out_bytes;
|
||||
}
|
||||
}
|
||||
|
||||
// Plan key encoding: rank (1 bit) | type (4 bits) | dims
|
||||
static int64_t plan_key_1d(int n, cufftType type) { return (int64_t(n) << 5) | (int64_t(type) << 1) | 0; }
|
||||
|
||||
static int64_t plan_key_2d(int rows, int cols, cufftType type) {
|
||||
return (int64_t(rows) << 35) | (int64_t(cols) << 5) | (int64_t(type) << 1) | 1;
|
||||
}
|
||||
|
||||
cufftHandle get_plan_1d(int n, cufftType type) {
|
||||
int64_t key = plan_key_1d(n, type);
|
||||
auto it = plans_.find(key);
|
||||
if (it != plans_.end()) return it->second;
|
||||
|
||||
cufftHandle plan;
|
||||
EIGEN_CUFFT_CHECK(cufftPlan1d(&plan, n, type, /*batch=*/1));
|
||||
EIGEN_CUFFT_CHECK(cufftSetStream(plan, stream_));
|
||||
plans_[key] = plan;
|
||||
return plan;
|
||||
}
|
||||
|
||||
cufftHandle get_plan_2d(int rows, int cols, cufftType type) {
|
||||
int64_t key = plan_key_2d(rows, cols, type);
|
||||
auto it = plans_.find(key);
|
||||
if (it != plans_.end()) return it->second;
|
||||
|
||||
// cuFFT uses row-major (C order) for 2D: first dim = rows, second = cols.
|
||||
// Eigen matrices are column-major, so we pass (cols, rows) to cuFFT
|
||||
// to get the correct 2D transform.
|
||||
cufftHandle plan;
|
||||
EIGEN_CUFFT_CHECK(cufftPlan2d(&plan, cols, rows, type));
|
||||
EIGEN_CUFFT_CHECK(cufftSetStream(plan, stream_));
|
||||
plans_[key] = plan;
|
||||
return plan;
|
||||
}
|
||||
|
||||
// Scale complex array on device using cuBLAS scal.
|
||||
void scale_device(Complex* d_ptr, int n, Scalar alpha) { scale_complex(cublas_, d_ptr, n, alpha); }
|
||||
|
||||
// Scale real array on device using cuBLAS scal.
|
||||
void scale_device_real(Scalar* d_ptr, int n, Scalar alpha) { scale_real(cublas_, d_ptr, n, alpha); }
|
||||
|
||||
// Type-dispatched cuBLAS scal wrappers (C++14 compatible).
|
||||
static void scale_complex(cublasHandle_t h, std::complex<float>* p, int n, float a) {
|
||||
EIGEN_CUBLAS_CHECK(cublasCsscal(h, n, &a, reinterpret_cast<cuComplex*>(p), 1));
|
||||
}
|
||||
static void scale_complex(cublasHandle_t h, std::complex<double>* p, int n, double a) {
|
||||
EIGEN_CUBLAS_CHECK(cublasZdscal(h, n, &a, reinterpret_cast<cuDoubleComplex*>(p), 1));
|
||||
}
|
||||
static void scale_real(cublasHandle_t h, float* p, int n, float a) {
|
||||
EIGEN_CUBLAS_CHECK(cublasSscal(h, n, &a, p, 1));
|
||||
}
|
||||
static void scale_real(cublasHandle_t h, double* p, int n, double a) {
|
||||
EIGEN_CUBLAS_CHECK(cublasDscal(h, n, &a, p, 1));
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace Eigen
|
||||
|
||||
#endif // EIGEN_GPU_FFT_H
|
||||
@@ -179,7 +179,10 @@ class GpuQR {
|
||||
|
||||
/** Solve A * X = B via QR: X = R^{-1} * Q^H * B (least-squares for m >= n).
|
||||
* Uses ormqr (apply Q^H) + trsm (solve R), without forming Q explicitly.
|
||||
* Requires m >= n (overdetermined or square). Underdetermined not supported. */
|
||||
* Requires m >= n (overdetermined or square). Underdetermined not supported.
|
||||
*
|
||||
* TODO: Add device-side accessor for the R factor (and Q application) as
|
||||
* DeviceMatrix, so users can chain GPU operations without host round-trips. */
|
||||
template <typename Rhs>
|
||||
PlainMatrix solve(const MatrixBase<Rhs>& B) const {
|
||||
sync_info();
|
||||
|
||||
@@ -143,6 +143,10 @@ class GpuSVD {
|
||||
Index rows() const { return transposed_ ? n_ : m_; }
|
||||
Index cols() const { return transposed_ ? m_ : n_; }
|
||||
|
||||
// TODO: Add device-side accessors (deviceU(), deviceVT(), deviceSingularValues())
|
||||
// returning DeviceMatrix views of the internal buffers, so users can chain
|
||||
// GPU operations without round-tripping through host memory.
|
||||
|
||||
/** Singular values (always available). Downloads from device on each call. */
|
||||
RealVector singularValues() const {
|
||||
sync_info();
|
||||
|
||||
321
Eigen/src/GPU/GpuSparseContext.h
Normal file
321
Eigen/src/GPU/GpuSparseContext.h
Normal file
@@ -0,0 +1,321 @@
|
||||
// This file is part of Eigen, a lightweight C++ template library
|
||||
// for linear algebra.
|
||||
//
|
||||
// Copyright (C) 2026 Rasmus Munk Larsen <rmlarsen@gmail.com>
|
||||
//
|
||||
// This Source Code Form is subject to the terms of the Mozilla
|
||||
// Public License v. 2.0. If a copy of the MPL was not distributed
|
||||
// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
|
||||
// GPU sparse matrix-vector multiply (SpMV) and sparse matrix-dense matrix
|
||||
// multiply (SpMM) via cuSPARSE.
|
||||
//
|
||||
// GpuSparseContext manages a cuSPARSE handle and device buffers. It accepts
|
||||
// Eigen SparseMatrix<Scalar, ColMajor> (CSC) and performs SpMV/SpMM on the
|
||||
// GPU. RowMajor input is implicitly converted to ColMajor.
|
||||
//
|
||||
// Usage:
|
||||
// GpuSparseContext<double> ctx;
|
||||
// VectorXd y = ctx.multiply(A, x); // y = A * x
|
||||
// ctx.multiply(A, x, y, 2.0, 1.0); // y = 2*A*x + y
|
||||
// VectorXd z = ctx.multiplyT(A, x); // z = A^T * x
|
||||
// MatrixXd Y = ctx.multiplyMat(A, X); // Y = A * X (multiple RHS)
|
||||
|
||||
#ifndef EIGEN_GPU_SPARSE_CONTEXT_H
|
||||
#define EIGEN_GPU_SPARSE_CONTEXT_H
|
||||
|
||||
// IWYU pragma: private
|
||||
#include "./InternalHeaderCheck.h"
|
||||
|
||||
#include "./CuSparseSupport.h"
|
||||
|
||||
namespace Eigen {
|
||||
|
||||
template <typename Scalar_>
|
||||
class GpuSparseContext {
|
||||
public:
|
||||
using Scalar = Scalar_;
|
||||
using RealScalar = typename NumTraits<Scalar>::Real;
|
||||
using StorageIndex = int;
|
||||
using SpMat = SparseMatrix<Scalar, ColMajor, StorageIndex>;
|
||||
using DenseVector = Matrix<Scalar, Dynamic, 1>;
|
||||
using DenseMatrix = Matrix<Scalar, Dynamic, Dynamic, ColMajor>;
|
||||
|
||||
GpuSparseContext() {
|
||||
EIGEN_CUDA_RUNTIME_CHECK(cudaStreamCreate(&stream_));
|
||||
EIGEN_CUSPARSE_CHECK(cusparseCreate(&handle_));
|
||||
EIGEN_CUSPARSE_CHECK(cusparseSetStream(handle_, stream_));
|
||||
}
|
||||
|
||||
~GpuSparseContext() {
|
||||
destroy_descriptors();
|
||||
if (handle_) (void)cusparseDestroy(handle_);
|
||||
if (stream_) (void)cudaStreamDestroy(stream_);
|
||||
}
|
||||
|
||||
GpuSparseContext(const GpuSparseContext&) = delete;
|
||||
GpuSparseContext& operator=(const GpuSparseContext&) = delete;
|
||||
|
||||
// ---- SpMV: y = A * x -----------------------------------------------------
|
||||
|
||||
/** Compute y = A * x. Returns y as a new dense vector. */
|
||||
template <typename InputType, typename Rhs>
|
||||
DenseVector multiply(const SparseMatrixBase<InputType>& A, const MatrixBase<Rhs>& x) {
|
||||
const SpMat mat(A.derived());
|
||||
DenseVector y(mat.rows());
|
||||
y.setZero();
|
||||
multiply_impl(mat, x.derived(), y, Scalar(1), Scalar(0), CUSPARSE_OPERATION_NON_TRANSPOSE);
|
||||
return y;
|
||||
}
|
||||
|
||||
/** Compute y = alpha * op(A) * x + beta * y (in-place). */
|
||||
template <typename InputType, typename Rhs, typename Dest>
|
||||
void multiply(const SparseMatrixBase<InputType>& A, const MatrixBase<Rhs>& x, MatrixBase<Dest>& y,
|
||||
Scalar alpha = Scalar(1), Scalar beta = Scalar(0),
|
||||
cusparseOperation_t op = CUSPARSE_OPERATION_NON_TRANSPOSE) {
|
||||
const SpMat mat(A.derived());
|
||||
multiply_impl(mat, x.derived(), y.derived(), alpha, beta, op);
|
||||
}
|
||||
|
||||
// ---- SpMV transpose: y = A^T * x -----------------------------------------
|
||||
|
||||
/** Compute y = A^T * x. Returns y as a new dense vector. */
|
||||
template <typename InputType, typename Rhs>
|
||||
DenseVector multiplyT(const SparseMatrixBase<InputType>& A, const MatrixBase<Rhs>& x) {
|
||||
const SpMat mat(A.derived());
|
||||
DenseVector y(mat.cols());
|
||||
y.setZero();
|
||||
multiply_impl(mat, x.derived(), y, Scalar(1), Scalar(0), CUSPARSE_OPERATION_TRANSPOSE);
|
||||
return y;
|
||||
}
|
||||
|
||||
// ---- SpMM: Y = A * X (multiple RHS) --------------------------------------
|
||||
|
||||
/** Compute Y = A * X where X is a dense matrix (multiple RHS). Returns Y. */
|
||||
template <typename InputType, typename Rhs>
|
||||
DenseMatrix multiplyMat(const SparseMatrixBase<InputType>& A, const MatrixBase<Rhs>& X) {
|
||||
const SpMat mat(A.derived());
|
||||
const DenseMatrix rhs(X.derived());
|
||||
eigen_assert(mat.cols() == rhs.rows());
|
||||
|
||||
const Index m = mat.rows();
|
||||
const Index n = rhs.cols();
|
||||
if (m == 0 || n == 0 || mat.nonZeros() == 0) return DenseMatrix::Zero(m, n);
|
||||
|
||||
DenseMatrix Y = DenseMatrix::Zero(m, n);
|
||||
spmm_impl(mat, rhs, Y, Scalar(1), Scalar(0), CUSPARSE_OPERATION_NON_TRANSPOSE);
|
||||
return Y;
|
||||
}
|
||||
|
||||
// ---- Accessors ------------------------------------------------------------
|
||||
|
||||
cudaStream_t stream() const { return stream_; }
|
||||
|
||||
private:
|
||||
cudaStream_t stream_ = nullptr;
|
||||
cusparseHandle_t handle_ = nullptr;
|
||||
|
||||
// Cached device buffers (grow-only).
|
||||
internal::DeviceBuffer d_outerPtr_;
|
||||
internal::DeviceBuffer d_innerIdx_;
|
||||
internal::DeviceBuffer d_values_;
|
||||
internal::DeviceBuffer d_x_;
|
||||
internal::DeviceBuffer d_y_;
|
||||
internal::DeviceBuffer d_workspace_;
|
||||
size_t d_outerPtr_size_ = 0;
|
||||
size_t d_innerIdx_size_ = 0;
|
||||
size_t d_values_size_ = 0;
|
||||
size_t d_x_size_ = 0;
|
||||
size_t d_y_size_ = 0;
|
||||
size_t d_workspace_size_ = 0;
|
||||
|
||||
// Cached cuSPARSE descriptors.
|
||||
cusparseSpMatDescr_t spmat_desc_ = nullptr;
|
||||
Index cached_rows_ = -1;
|
||||
Index cached_cols_ = -1;
|
||||
Index cached_nnz_ = -1;
|
||||
|
||||
// ---- SpMV implementation --------------------------------------------------
|
||||
|
||||
template <typename RhsDerived, typename DestDerived>
|
||||
void multiply_impl(const SpMat& A, const RhsDerived& x, DestDerived& y, Scalar alpha, Scalar beta,
|
||||
cusparseOperation_t op) {
|
||||
eigen_assert(A.isCompressed());
|
||||
|
||||
const Index m = A.rows();
|
||||
const Index n = A.cols();
|
||||
const Index nnz = A.nonZeros();
|
||||
const Index x_size = (op == CUSPARSE_OPERATION_NON_TRANSPOSE) ? n : m;
|
||||
const Index y_size = (op == CUSPARSE_OPERATION_NON_TRANSPOSE) ? m : n;
|
||||
|
||||
eigen_assert(x.size() == x_size);
|
||||
eigen_assert(y.size() == y_size);
|
||||
|
||||
if (m == 0 || n == 0 || nnz == 0) {
|
||||
if (beta == Scalar(0))
|
||||
y.setZero();
|
||||
else
|
||||
y *= beta;
|
||||
return;
|
||||
}
|
||||
|
||||
// Upload sparse matrix to device.
|
||||
upload_sparse(A);
|
||||
|
||||
// Upload x to device.
|
||||
ensure_buffer(d_x_, d_x_size_, static_cast<size_t>(x_size) * sizeof(Scalar));
|
||||
const DenseVector x_tmp(x);
|
||||
EIGEN_CUDA_RUNTIME_CHECK(
|
||||
cudaMemcpyAsync(d_x_.ptr, x_tmp.data(), x_size * sizeof(Scalar), cudaMemcpyHostToDevice, stream_));
|
||||
|
||||
// Upload y to device (for beta != 0).
|
||||
ensure_buffer(d_y_, d_y_size_, static_cast<size_t>(y_size) * sizeof(Scalar));
|
||||
if (beta != Scalar(0)) {
|
||||
const DenseVector y_tmp(y);
|
||||
EIGEN_CUDA_RUNTIME_CHECK(
|
||||
cudaMemcpyAsync(d_y_.ptr, y_tmp.data(), y_size * sizeof(Scalar), cudaMemcpyHostToDevice, stream_));
|
||||
}
|
||||
|
||||
// Create dense vector descriptors.
|
||||
constexpr cudaDataType_t dtype = internal::cuda_data_type<Scalar>::value;
|
||||
cusparseDnVecDescr_t x_desc = nullptr, y_desc = nullptr;
|
||||
EIGEN_CUSPARSE_CHECK(cusparseCreateDnVec(&x_desc, x_size, d_x_.ptr, dtype));
|
||||
EIGEN_CUSPARSE_CHECK(cusparseCreateDnVec(&y_desc, y_size, d_y_.ptr, dtype));
|
||||
|
||||
// Query workspace size.
|
||||
size_t ws_size = 0;
|
||||
EIGEN_CUSPARSE_CHECK(cusparseSpMV_bufferSize(handle_, op, &alpha, spmat_desc_, x_desc, &beta, y_desc, dtype,
|
||||
CUSPARSE_SPMV_ALG_DEFAULT, &ws_size));
|
||||
ensure_buffer(d_workspace_, d_workspace_size_, ws_size);
|
||||
|
||||
// Execute SpMV.
|
||||
EIGEN_CUSPARSE_CHECK(cusparseSpMV(handle_, op, &alpha, spmat_desc_, x_desc, &beta, y_desc, dtype,
|
||||
CUSPARSE_SPMV_ALG_DEFAULT, d_workspace_.ptr));
|
||||
|
||||
// Download result.
|
||||
EIGEN_CUDA_RUNTIME_CHECK(
|
||||
cudaMemcpyAsync(y.data(), d_y_.ptr, y_size * sizeof(Scalar), cudaMemcpyDeviceToHost, stream_));
|
||||
EIGEN_CUDA_RUNTIME_CHECK(cudaStreamSynchronize(stream_));
|
||||
|
||||
(void)cusparseDestroyDnVec(x_desc);
|
||||
(void)cusparseDestroyDnVec(y_desc);
|
||||
}
|
||||
|
||||
// ---- SpMM implementation --------------------------------------------------
|
||||
|
||||
void spmm_impl(const SpMat& A, const DenseMatrix& X, DenseMatrix& Y, Scalar alpha, Scalar beta,
|
||||
cusparseOperation_t op) {
|
||||
eigen_assert(A.isCompressed());
|
||||
|
||||
const Index m = A.rows();
|
||||
const Index n = X.cols();
|
||||
const Index k = A.cols();
|
||||
const Index nnz = A.nonZeros();
|
||||
|
||||
if (m == 0 || n == 0 || k == 0 || nnz == 0) {
|
||||
if (beta == Scalar(0))
|
||||
Y.setZero();
|
||||
else
|
||||
Y *= beta;
|
||||
return;
|
||||
}
|
||||
|
||||
upload_sparse(A);
|
||||
|
||||
// Upload X to device.
|
||||
const size_t x_bytes = static_cast<size_t>(k) * static_cast<size_t>(n) * sizeof(Scalar);
|
||||
const size_t y_bytes = static_cast<size_t>(m) * static_cast<size_t>(n) * sizeof(Scalar);
|
||||
ensure_buffer(d_x_, d_x_size_, x_bytes);
|
||||
ensure_buffer(d_y_, d_y_size_, y_bytes);
|
||||
EIGEN_CUDA_RUNTIME_CHECK(cudaMemcpyAsync(d_x_.ptr, X.data(), x_bytes, cudaMemcpyHostToDevice, stream_));
|
||||
if (beta != Scalar(0)) {
|
||||
EIGEN_CUDA_RUNTIME_CHECK(cudaMemcpyAsync(d_y_.ptr, Y.data(), y_bytes, cudaMemcpyHostToDevice, stream_));
|
||||
}
|
||||
|
||||
// Create dense matrix descriptors.
|
||||
constexpr cudaDataType_t dtype = internal::cuda_data_type<Scalar>::value;
|
||||
cusparseDnMatDescr_t x_desc = nullptr, y_desc = nullptr;
|
||||
// Eigen is column-major, so ld = rows.
|
||||
EIGEN_CUSPARSE_CHECK(cusparseCreateDnMat(&x_desc, k, n, k, d_x_.ptr, dtype, CUSPARSE_ORDER_COL));
|
||||
EIGEN_CUSPARSE_CHECK(cusparseCreateDnMat(&y_desc, m, n, m, d_y_.ptr, dtype, CUSPARSE_ORDER_COL));
|
||||
|
||||
// Query workspace.
|
||||
size_t ws_size = 0;
|
||||
EIGEN_CUSPARSE_CHECK(cusparseSpMM_bufferSize(handle_, op, CUSPARSE_OPERATION_NON_TRANSPOSE, &alpha, spmat_desc_,
|
||||
x_desc, &beta, y_desc, dtype, CUSPARSE_SPMM_ALG_DEFAULT, &ws_size));
|
||||
ensure_buffer(d_workspace_, d_workspace_size_, ws_size);
|
||||
|
||||
// Execute SpMM.
|
||||
EIGEN_CUSPARSE_CHECK(cusparseSpMM(handle_, op, CUSPARSE_OPERATION_NON_TRANSPOSE, &alpha, spmat_desc_, x_desc, &beta,
|
||||
y_desc, dtype, CUSPARSE_SPMM_ALG_DEFAULT, d_workspace_.ptr));
|
||||
|
||||
// Download result.
|
||||
EIGEN_CUDA_RUNTIME_CHECK(cudaMemcpyAsync(Y.data(), d_y_.ptr, y_bytes, cudaMemcpyDeviceToHost, stream_));
|
||||
EIGEN_CUDA_RUNTIME_CHECK(cudaStreamSynchronize(stream_));
|
||||
|
||||
(void)cusparseDestroyDnMat(x_desc);
|
||||
(void)cusparseDestroyDnMat(y_desc);
|
||||
}
|
||||
|
||||
// ---- Helpers --------------------------------------------------------------
|
||||
|
||||
void upload_sparse(const SpMat& A) {
|
||||
const Index m = A.rows();
|
||||
const Index n = A.cols();
|
||||
const Index nnz = A.nonZeros();
|
||||
|
||||
const size_t outer_bytes = static_cast<size_t>(n + 1) * sizeof(StorageIndex);
|
||||
const size_t inner_bytes = static_cast<size_t>(nnz) * sizeof(StorageIndex);
|
||||
const size_t val_bytes = static_cast<size_t>(nnz) * sizeof(Scalar);
|
||||
|
||||
ensure_buffer(d_outerPtr_, d_outerPtr_size_, outer_bytes);
|
||||
ensure_buffer(d_innerIdx_, d_innerIdx_size_, inner_bytes);
|
||||
ensure_buffer(d_values_, d_values_size_, val_bytes);
|
||||
|
||||
EIGEN_CUDA_RUNTIME_CHECK(
|
||||
cudaMemcpyAsync(d_outerPtr_.ptr, A.outerIndexPtr(), outer_bytes, cudaMemcpyHostToDevice, stream_));
|
||||
EIGEN_CUDA_RUNTIME_CHECK(
|
||||
cudaMemcpyAsync(d_innerIdx_.ptr, A.innerIndexPtr(), inner_bytes, cudaMemcpyHostToDevice, stream_));
|
||||
EIGEN_CUDA_RUNTIME_CHECK(cudaMemcpyAsync(d_values_.ptr, A.valuePtr(), val_bytes, cudaMemcpyHostToDevice, stream_));
|
||||
|
||||
// Recreate descriptor if shape changed.
|
||||
if (m != cached_rows_ || n != cached_cols_ || nnz != cached_nnz_) {
|
||||
destroy_descriptors();
|
||||
|
||||
constexpr cusparseIndexType_t idx_type = (sizeof(StorageIndex) == 4) ? CUSPARSE_INDEX_32I : CUSPARSE_INDEX_64I;
|
||||
constexpr cudaDataType_t val_type = internal::cuda_data_type<Scalar>::value;
|
||||
|
||||
// ColMajor → CSC. outerIndexPtr = col offsets, innerIndexPtr = row indices.
|
||||
EIGEN_CUSPARSE_CHECK(cusparseCreateCsc(&spmat_desc_, m, n, nnz, d_outerPtr_.ptr, d_innerIdx_.ptr, d_values_.ptr,
|
||||
idx_type, idx_type, CUSPARSE_INDEX_BASE_ZERO, val_type));
|
||||
cached_rows_ = m;
|
||||
cached_cols_ = n;
|
||||
cached_nnz_ = nnz;
|
||||
} else {
|
||||
// Same shape — just update pointers.
|
||||
EIGEN_CUSPARSE_CHECK(cusparseCscSetPointers(spmat_desc_, d_outerPtr_.ptr, d_innerIdx_.ptr, d_values_.ptr));
|
||||
}
|
||||
}
|
||||
|
||||
void destroy_descriptors() {
|
||||
if (spmat_desc_) {
|
||||
(void)cusparseDestroySpMat(spmat_desc_);
|
||||
spmat_desc_ = nullptr;
|
||||
}
|
||||
cached_rows_ = -1;
|
||||
cached_cols_ = -1;
|
||||
cached_nnz_ = -1;
|
||||
}
|
||||
|
||||
void ensure_buffer(internal::DeviceBuffer& buf, size_t& current_size, size_t needed) {
|
||||
if (needed > current_size) {
|
||||
if (buf.ptr) EIGEN_CUDA_RUNTIME_CHECK(cudaStreamSynchronize(stream_));
|
||||
buf = internal::DeviceBuffer(needed);
|
||||
current_size = needed;
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace Eigen
|
||||
|
||||
#endif // EIGEN_GPU_SPARSE_CONTEXT_H
|
||||
62
Eigen/src/GPU/GpuSparseLDLT.h
Normal file
62
Eigen/src/GPU/GpuSparseLDLT.h
Normal file
@@ -0,0 +1,62 @@
|
||||
// This file is part of Eigen, a lightweight C++ template library
|
||||
// for linear algebra.
|
||||
//
|
||||
// Copyright (C) 2026 Rasmus Munk Larsen <rmlarsen@gmail.com>
|
||||
//
|
||||
// This Source Code Form is subject to the terms of the Mozilla
|
||||
// Public License v. 2.0. If a copy of the MPL was not distributed
|
||||
// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
|
||||
// GPU sparse LDL^T / LDL^H factorization via cuDSS.
|
||||
//
|
||||
// For symmetric indefinite (or Hermitian indefinite) sparse matrices.
|
||||
// Same three-phase workflow as GpuSparseLLT.
|
||||
//
|
||||
// Usage:
|
||||
// GpuSparseLDLT<double> ldlt(A); // analyze + factorize
|
||||
// VectorXd x = ldlt.solve(b); // solve
|
||||
|
||||
#ifndef EIGEN_GPU_SPARSE_LDLT_H
|
||||
#define EIGEN_GPU_SPARSE_LDLT_H
|
||||
|
||||
// IWYU pragma: private
|
||||
#include "./InternalHeaderCheck.h"
|
||||
|
||||
#include "./GpuSparseSolverBase.h"
|
||||
|
||||
namespace Eigen {
|
||||
|
||||
/** GPU sparse LDL^T factorization (symmetric indefinite / Hermitian indefinite).
|
||||
*
|
||||
* Wraps cuDSS with CUDSS_MTYPE_SYMMETRIC (real) or CUDSS_MTYPE_HERMITIAN (complex).
|
||||
* Uses pivoting for numerical stability.
|
||||
*
|
||||
* \tparam Scalar_ float, double, complex<float>, or complex<double>
|
||||
* \tparam UpLo_ Lower (default) or Upper — which triangle of A is stored
|
||||
*/
|
||||
template <typename Scalar_, int UpLo_ = Lower>
|
||||
class GpuSparseLDLT : public internal::GpuSparseSolverBase<Scalar_, GpuSparseLDLT<Scalar_, UpLo_>> {
|
||||
using Base = internal::GpuSparseSolverBase<Scalar_, GpuSparseLDLT>;
|
||||
friend Base;
|
||||
|
||||
public:
|
||||
using Scalar = Scalar_;
|
||||
enum { UpLo = UpLo_ };
|
||||
|
||||
GpuSparseLDLT() = default;
|
||||
|
||||
template <typename InputType>
|
||||
explicit GpuSparseLDLT(const SparseMatrixBase<InputType>& A) {
|
||||
this->compute(A);
|
||||
}
|
||||
|
||||
static constexpr bool needs_csr_conversion() { return false; }
|
||||
static constexpr cudssMatrixType_t cudss_matrix_type() { return internal::cudss_symmetric_type<Scalar>::value; }
|
||||
static constexpr cudssMatrixViewType_t cudss_matrix_view() {
|
||||
return internal::cudss_view_type<UpLo, ColMajor>::value;
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace Eigen
|
||||
|
||||
#endif // EIGEN_GPU_SPARSE_LDLT_H
|
||||
62
Eigen/src/GPU/GpuSparseLLT.h
Normal file
62
Eigen/src/GPU/GpuSparseLLT.h
Normal file
@@ -0,0 +1,62 @@
|
||||
// This file is part of Eigen, a lightweight C++ template library
|
||||
// for linear algebra.
|
||||
//
|
||||
// Copyright (C) 2026 Rasmus Munk Larsen <rmlarsen@gmail.com>
|
||||
//
|
||||
// This Source Code Form is subject to the terms of the Mozilla
|
||||
// Public License v. 2.0. If a copy of the MPL was not distributed
|
||||
// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
|
||||
// GPU sparse Cholesky (LL^T / LL^H) via cuDSS.
|
||||
//
|
||||
// Usage:
|
||||
// GpuSparseLLT<double> llt(A); // analyze + factorize
|
||||
// VectorXd x = llt.solve(b); // solve
|
||||
// llt.analyzePattern(A); // or separate phases
|
||||
// llt.factorize(A_new); // reuse symbolic analysis
|
||||
|
||||
#ifndef EIGEN_GPU_SPARSE_LLT_H
|
||||
#define EIGEN_GPU_SPARSE_LLT_H
|
||||
|
||||
// IWYU pragma: private
|
||||
#include "./InternalHeaderCheck.h"
|
||||
|
||||
#include "./GpuSparseSolverBase.h"
|
||||
|
||||
namespace Eigen {
|
||||
|
||||
/** GPU sparse Cholesky factorization (LL^T for real, LL^H for complex).
|
||||
*
|
||||
* Wraps cuDSS with CUDSS_MTYPE_SPD (real) or CUDSS_MTYPE_HPD (complex).
|
||||
* Accepts ColMajor SparseMatrix (CSC), reinterpreted as CSR with swapped
|
||||
* triangle view for zero-copy upload.
|
||||
*
|
||||
* \tparam Scalar_ float, double, complex<float>, or complex<double>
|
||||
* \tparam UpLo_ Lower (default) or Upper — which triangle of A is stored
|
||||
*/
|
||||
template <typename Scalar_, int UpLo_ = Lower>
|
||||
class GpuSparseLLT : public internal::GpuSparseSolverBase<Scalar_, GpuSparseLLT<Scalar_, UpLo_>> {
|
||||
using Base = internal::GpuSparseSolverBase<Scalar_, GpuSparseLLT>;
|
||||
friend Base;
|
||||
|
||||
public:
|
||||
using Scalar = Scalar_;
|
||||
enum { UpLo = UpLo_ };
|
||||
|
||||
GpuSparseLLT() = default;
|
||||
|
||||
template <typename InputType>
|
||||
explicit GpuSparseLLT(const SparseMatrixBase<InputType>& A) {
|
||||
this->compute(A);
|
||||
}
|
||||
|
||||
static constexpr bool needs_csr_conversion() { return false; }
|
||||
static constexpr cudssMatrixType_t cudss_matrix_type() { return internal::cudss_spd_type<Scalar>::value; }
|
||||
static constexpr cudssMatrixViewType_t cudss_matrix_view() {
|
||||
return internal::cudss_view_type<UpLo, ColMajor>::value;
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace Eigen
|
||||
|
||||
#endif // EIGEN_GPU_SPARSE_LLT_H
|
||||
59
Eigen/src/GPU/GpuSparseLU.h
Normal file
59
Eigen/src/GPU/GpuSparseLU.h
Normal file
@@ -0,0 +1,59 @@
|
||||
// This file is part of Eigen, a lightweight C++ template library
|
||||
// for linear algebra.
|
||||
//
|
||||
// Copyright (C) 2026 Rasmus Munk Larsen <rmlarsen@gmail.com>
|
||||
//
|
||||
// This Source Code Form is subject to the terms of the Mozilla
|
||||
// Public License v. 2.0. If a copy of the MPL was not distributed
|
||||
// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
|
||||
// GPU sparse LU factorization via cuDSS.
|
||||
//
|
||||
// For general (non-symmetric) sparse matrices. Uses pivoting.
|
||||
// Same three-phase workflow as GpuSparseLLT.
|
||||
//
|
||||
// Usage:
|
||||
// GpuSparseLU<double> lu(A); // analyze + factorize
|
||||
// VectorXd x = lu.solve(b); // solve
|
||||
|
||||
#ifndef EIGEN_GPU_SPARSE_LU_H
|
||||
#define EIGEN_GPU_SPARSE_LU_H
|
||||
|
||||
// IWYU pragma: private
|
||||
#include "./InternalHeaderCheck.h"
|
||||
|
||||
#include "./GpuSparseSolverBase.h"
|
||||
|
||||
namespace Eigen {
|
||||
|
||||
/** GPU sparse LU factorization (general matrices).
|
||||
*
|
||||
* Wraps cuDSS with CUDSS_MTYPE_GENERAL and CUDSS_MVIEW_FULL.
|
||||
* Accepts ColMajor SparseMatrix (CSC); internally converts to RowMajor
|
||||
* CSR since cuDSS requires CSR input.
|
||||
*
|
||||
* \tparam Scalar_ float, double, complex<float>, or complex<double>
|
||||
*/
|
||||
template <typename Scalar_>
|
||||
class GpuSparseLU : public internal::GpuSparseSolverBase<Scalar_, GpuSparseLU<Scalar_>> {
|
||||
using Base = internal::GpuSparseSolverBase<Scalar_, GpuSparseLU>;
|
||||
friend Base;
|
||||
|
||||
public:
|
||||
using Scalar = Scalar_;
|
||||
|
||||
GpuSparseLU() = default;
|
||||
|
||||
template <typename InputType>
|
||||
explicit GpuSparseLU(const SparseMatrixBase<InputType>& A) {
|
||||
this->compute(A);
|
||||
}
|
||||
|
||||
static constexpr bool needs_csr_conversion() { return true; }
|
||||
static constexpr cudssMatrixType_t cudss_matrix_type() { return CUDSS_MTYPE_GENERAL; }
|
||||
static constexpr cudssMatrixViewType_t cudss_matrix_view() { return CUDSS_MVIEW_FULL; }
|
||||
};
|
||||
|
||||
} // namespace Eigen
|
||||
|
||||
#endif // EIGEN_GPU_SPARSE_LU_H
|
||||
356
Eigen/src/GPU/GpuSparseSolverBase.h
Normal file
356
Eigen/src/GPU/GpuSparseSolverBase.h
Normal file
@@ -0,0 +1,356 @@
|
||||
// This file is part of Eigen, a lightweight C++ template library
|
||||
// for linear algebra.
|
||||
//
|
||||
// Copyright (C) 2026 Rasmus Munk Larsen <rmlarsen@gmail.com>
|
||||
//
|
||||
// This Source Code Form is subject to the terms of the Mozilla
|
||||
// Public License v. 2.0. If a copy of the MPL was not distributed
|
||||
// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
|
||||
// Common base for GPU sparse direct solvers (LLT, LDLT, LU) via cuDSS.
|
||||
//
|
||||
// All three solver types share the same three-phase workflow
|
||||
// (analyzePattern → factorize → solve) and differ only in the
|
||||
// cudssMatrixType_t and cudssMatrixViewType_t passed to cuDSS.
|
||||
// This CRTP base implements the entire workflow; derived classes
|
||||
// provide the matrix type/view via static constexpr members.
|
||||
|
||||
#ifndef EIGEN_GPU_SPARSE_SOLVER_BASE_H
|
||||
#define EIGEN_GPU_SPARSE_SOLVER_BASE_H
|
||||
|
||||
// IWYU pragma: private
|
||||
#include "./InternalHeaderCheck.h"
|
||||
|
||||
#include "./CuDssSupport.h"
|
||||
|
||||
namespace Eigen {
|
||||
namespace internal {
|
||||
|
||||
/** CRTP base for GPU sparse direct solvers.
|
||||
*
|
||||
* \tparam Scalar_ Element type (passed explicitly to avoid incomplete-type issues with CRTP).
|
||||
* \tparam Derived The concrete solver class (GpuSparseLLT, GpuSparseLDLT, GpuSparseLU).
|
||||
* Must provide:
|
||||
* - `static constexpr cudssMatrixType_t cudss_matrix_type()`
|
||||
* - `static constexpr cudssMatrixViewType_t cudss_matrix_view()`
|
||||
*/
|
||||
template <typename Scalar_, typename Derived>
|
||||
class GpuSparseSolverBase {
|
||||
public:
|
||||
using Scalar = Scalar_;
|
||||
using RealScalar = typename NumTraits<Scalar>::Real;
|
||||
using StorageIndex = int;
|
||||
using SpMat = SparseMatrix<Scalar, ColMajor, StorageIndex>;
|
||||
using CsrMat = SparseMatrix<Scalar, RowMajor, StorageIndex>;
|
||||
using DenseVector = Matrix<Scalar, Dynamic, 1>;
|
||||
using DenseMatrix = Matrix<Scalar, Dynamic, Dynamic, ColMajor>;
|
||||
|
||||
GpuSparseSolverBase() { init_context(); }
|
||||
|
||||
~GpuSparseSolverBase() {
|
||||
destroy_cudss_objects();
|
||||
if (handle_) (void)cudssDestroy(handle_);
|
||||
if (stream_) (void)cudaStreamDestroy(stream_);
|
||||
}
|
||||
|
||||
GpuSparseSolverBase(const GpuSparseSolverBase&) = delete;
|
||||
GpuSparseSolverBase& operator=(const GpuSparseSolverBase&) = delete;
|
||||
|
||||
// ---- Configuration --------------------------------------------------------
|
||||
|
||||
/** Set the fill-reducing ordering algorithm. Must be called before compute/analyzePattern. */
|
||||
void setOrdering(GpuSparseOrdering ordering) { ordering_ = ordering; }
|
||||
|
||||
// ---- Factorization --------------------------------------------------------
|
||||
|
||||
/** Symbolic analysis + numeric factorization. */
|
||||
template <typename InputType>
|
||||
Derived& compute(const SparseMatrixBase<InputType>& A) {
|
||||
analyzePattern(A);
|
||||
if (info_ == Success) {
|
||||
factorize(A);
|
||||
}
|
||||
return derived();
|
||||
}
|
||||
|
||||
/** Symbolic analysis only. Uploads sparsity structure to device.
|
||||
* This phase is synchronous (blocks until complete). */
|
||||
template <typename InputType>
|
||||
Derived& analyzePattern(const SparseMatrixBase<InputType>& A) {
|
||||
const SpMat csc(A.derived());
|
||||
eigen_assert(csc.rows() == csc.cols() && "GpuSparseSolver requires a square matrix");
|
||||
eigen_assert(csc.isCompressed() && "GpuSparseSolver requires a compressed sparse matrix");
|
||||
|
||||
n_ = csc.rows();
|
||||
info_ = InvalidInput;
|
||||
analysis_done_ = false;
|
||||
|
||||
if (n_ == 0) {
|
||||
nnz_ = 0;
|
||||
info_ = Success;
|
||||
analysis_done_ = true;
|
||||
return derived();
|
||||
}
|
||||
|
||||
// For symmetric solvers, ColMajor CSC can be reinterpreted as CSR with
|
||||
// swapped triangle view (zero copy). For general solvers, we must convert
|
||||
// to actual RowMajor CSR so cuDSS sees the correct matrix, not A^T.
|
||||
if (Derived::needs_csr_conversion()) {
|
||||
const CsrMat csr(csc);
|
||||
nnz_ = csr.nonZeros();
|
||||
upload_csr(csr);
|
||||
} else {
|
||||
nnz_ = csc.nonZeros();
|
||||
upload_csr_from_csc(csc);
|
||||
}
|
||||
create_cudss_matrix();
|
||||
apply_ordering_config();
|
||||
|
||||
if (data_) EIGEN_CUDSS_CHECK(cudssDataDestroy(handle_, data_));
|
||||
EIGEN_CUDSS_CHECK(cudssDataCreate(handle_, &data_));
|
||||
|
||||
create_placeholder_dense();
|
||||
|
||||
EIGEN_CUDSS_CHECK(cudssExecute(handle_, CUDSS_PHASE_ANALYSIS, config_, data_, d_A_cudss_, d_x_cudss_, d_b_cudss_));
|
||||
|
||||
analysis_done_ = true;
|
||||
info_ = Success;
|
||||
return derived();
|
||||
}
|
||||
|
||||
/** Numeric factorization using the symbolic analysis from analyzePattern.
|
||||
*
|
||||
* \warning The sparsity pattern (outerIndexPtr, innerIndexPtr) must be
|
||||
* identical to the one passed to analyzePattern(). Only the numerical
|
||||
* values may change. Passing a different pattern is undefined behavior.
|
||||
* This matches the contract of CHOLMOD, UMFPACK, and cuDSS's own API.
|
||||
*
|
||||
* This phase is asynchronous — info() lazily synchronizes. */
|
||||
template <typename InputType>
|
||||
Derived& factorize(const SparseMatrixBase<InputType>& A) {
|
||||
eigen_assert(analysis_done_ && "factorize() requires analyzePattern() first");
|
||||
|
||||
if (n_ == 0) {
|
||||
info_ = Success;
|
||||
return derived();
|
||||
}
|
||||
|
||||
// Convert to the same format used in analyzePattern.
|
||||
// Both temporaries must outlive the async memcpy (pageable H2D is actually
|
||||
// synchronous w.r.t. the host, but keep them alive for clarity).
|
||||
const SpMat csc(A.derived());
|
||||
eigen_assert(csc.rows() == n_ && csc.cols() == n_);
|
||||
|
||||
const Scalar* value_ptr;
|
||||
Index value_nnz;
|
||||
CsrMat csr_tmp;
|
||||
if (Derived::needs_csr_conversion()) {
|
||||
csr_tmp = CsrMat(csc);
|
||||
value_ptr = csr_tmp.valuePtr();
|
||||
value_nnz = csr_tmp.nonZeros();
|
||||
} else {
|
||||
value_ptr = csc.valuePtr();
|
||||
value_nnz = csc.nonZeros();
|
||||
}
|
||||
eigen_assert(value_nnz == nnz_);
|
||||
|
||||
EIGEN_CUDA_RUNTIME_CHECK(cudaMemcpyAsync(d_values_.ptr, value_ptr, static_cast<size_t>(nnz_) * sizeof(Scalar),
|
||||
cudaMemcpyHostToDevice, stream_));
|
||||
|
||||
EIGEN_CUDSS_CHECK(cudssMatrixSetValues(d_A_cudss_, d_values_.ptr));
|
||||
|
||||
info_ = InvalidInput;
|
||||
info_synced_ = false;
|
||||
EIGEN_CUDSS_CHECK(
|
||||
cudssExecute(handle_, CUDSS_PHASE_FACTORIZATION, config_, data_, d_A_cudss_, d_x_cudss_, d_b_cudss_));
|
||||
|
||||
return derived();
|
||||
}
|
||||
|
||||
// ---- Solve ----------------------------------------------------------------
|
||||
|
||||
/** Solve A * X = B. Returns X as a dense matrix.
|
||||
* Supports single or multiple right-hand sides. */
|
||||
template <typename Rhs>
|
||||
DenseMatrix solve(const MatrixBase<Rhs>& B) const {
|
||||
sync_info();
|
||||
eigen_assert(info_ == Success && "GpuSparseSolver::solve requires a successful factorization");
|
||||
eigen_assert(B.rows() == n_);
|
||||
|
||||
const DenseMatrix rhs(B);
|
||||
const int64_t nrhs = static_cast<int64_t>(rhs.cols());
|
||||
|
||||
if (n_ == 0) return DenseMatrix(0, rhs.cols());
|
||||
|
||||
const size_t rhs_bytes = static_cast<size_t>(n_) * static_cast<size_t>(nrhs) * sizeof(Scalar);
|
||||
DeviceBuffer d_b(rhs_bytes);
|
||||
DeviceBuffer d_x(rhs_bytes);
|
||||
EIGEN_CUDA_RUNTIME_CHECK(cudaMemcpyAsync(d_b.ptr, rhs.data(), rhs_bytes, cudaMemcpyHostToDevice, stream_));
|
||||
|
||||
constexpr cudaDataType_t dtype = cuda_data_type<Scalar>::value;
|
||||
cudssMatrix_t b_cudss = nullptr, x_cudss = nullptr;
|
||||
EIGEN_CUDSS_CHECK(cudssMatrixCreateDn(&b_cudss, static_cast<int64_t>(n_), nrhs, static_cast<int64_t>(n_), d_b.ptr,
|
||||
dtype, CUDSS_LAYOUT_COL_MAJOR));
|
||||
EIGEN_CUDSS_CHECK(cudssMatrixCreateDn(&x_cudss, static_cast<int64_t>(n_), nrhs, static_cast<int64_t>(n_), d_x.ptr,
|
||||
dtype, CUDSS_LAYOUT_COL_MAJOR));
|
||||
|
||||
EIGEN_CUDSS_CHECK(cudssExecute(handle_, CUDSS_PHASE_SOLVE, config_, data_, d_A_cudss_, x_cudss, b_cudss));
|
||||
|
||||
DenseMatrix X(n_, rhs.cols());
|
||||
EIGEN_CUDA_RUNTIME_CHECK(cudaMemcpyAsync(X.data(), d_x.ptr, rhs_bytes, cudaMemcpyDeviceToHost, stream_));
|
||||
EIGEN_CUDA_RUNTIME_CHECK(cudaStreamSynchronize(stream_));
|
||||
|
||||
(void)cudssMatrixDestroy(b_cudss);
|
||||
(void)cudssMatrixDestroy(x_cudss);
|
||||
|
||||
return X;
|
||||
}
|
||||
|
||||
// ---- Accessors ------------------------------------------------------------
|
||||
|
||||
ComputationInfo info() const {
|
||||
sync_info();
|
||||
return info_;
|
||||
}
|
||||
Index rows() const { return n_; }
|
||||
Index cols() const { return n_; }
|
||||
|
||||
cudaStream_t stream() const { return stream_; }
|
||||
|
||||
protected:
|
||||
// ---- CUDA / cuDSS handles -------------------------------------------------
|
||||
cudaStream_t stream_ = nullptr;
|
||||
cudssHandle_t handle_ = nullptr;
|
||||
cudssConfig_t config_ = nullptr;
|
||||
cudssData_t data_ = nullptr;
|
||||
cudssMatrix_t d_A_cudss_ = nullptr;
|
||||
cudssMatrix_t d_x_cudss_ = nullptr;
|
||||
cudssMatrix_t d_b_cudss_ = nullptr;
|
||||
|
||||
// ---- Device buffers for CSR arrays ----------------------------------------
|
||||
DeviceBuffer d_rowPtr_;
|
||||
DeviceBuffer d_colIdx_;
|
||||
DeviceBuffer d_values_;
|
||||
|
||||
// ---- State ----------------------------------------------------------------
|
||||
Index n_ = 0;
|
||||
Index nnz_ = 0;
|
||||
ComputationInfo info_ = InvalidInput;
|
||||
bool info_synced_ = true;
|
||||
bool analysis_done_ = false;
|
||||
GpuSparseOrdering ordering_ = GpuSparseOrdering::AMD;
|
||||
|
||||
private:
|
||||
Derived& derived() { return static_cast<Derived&>(*this); }
|
||||
const Derived& derived() const { return static_cast<const Derived&>(*this); }
|
||||
|
||||
void init_context() {
|
||||
EIGEN_CUDA_RUNTIME_CHECK(cudaStreamCreate(&stream_));
|
||||
EIGEN_CUDSS_CHECK(cudssCreate(&handle_));
|
||||
EIGEN_CUDSS_CHECK(cudssSetStream(handle_, stream_));
|
||||
EIGEN_CUDSS_CHECK(cudssConfigCreate(&config_));
|
||||
}
|
||||
|
||||
void sync_info() const {
|
||||
if (!info_synced_) {
|
||||
EIGEN_CUDA_RUNTIME_CHECK(cudaStreamSynchronize(stream_));
|
||||
int cudss_info = 0;
|
||||
EIGEN_CUDSS_CHECK(cudssDataGet(handle_, data_, CUDSS_DATA_INFO, &cudss_info, sizeof(cudss_info), nullptr));
|
||||
auto* self = const_cast<GpuSparseSolverBase*>(this);
|
||||
self->info_ = (cudss_info == 0) ? Success : NumericalIssue;
|
||||
self->info_synced_ = true;
|
||||
}
|
||||
}
|
||||
|
||||
void destroy_cudss_objects() {
|
||||
if (d_A_cudss_) {
|
||||
(void)cudssMatrixDestroy(d_A_cudss_);
|
||||
d_A_cudss_ = nullptr;
|
||||
}
|
||||
if (d_x_cudss_) {
|
||||
(void)cudssMatrixDestroy(d_x_cudss_);
|
||||
d_x_cudss_ = nullptr;
|
||||
}
|
||||
if (d_b_cudss_) {
|
||||
(void)cudssMatrixDestroy(d_b_cudss_);
|
||||
d_b_cudss_ = nullptr;
|
||||
}
|
||||
if (data_) {
|
||||
(void)cudssDataDestroy(handle_, data_);
|
||||
data_ = nullptr;
|
||||
}
|
||||
if (config_) {
|
||||
(void)cudssConfigDestroy(config_);
|
||||
config_ = nullptr;
|
||||
}
|
||||
}
|
||||
|
||||
// Upload CSR from a RowMajor sparse matrix (native CSR).
|
||||
void upload_csr(const CsrMat& csr) { upload_compressed(csr.outerIndexPtr(), csr.innerIndexPtr(), csr.valuePtr()); }
|
||||
|
||||
// Upload CSC arrays reinterpreted as CSR (for symmetric matrices: CSC(A) = CSR(A^T) = CSR(A)).
|
||||
void upload_csr_from_csc(const SpMat& csc) {
|
||||
upload_compressed(csc.outerIndexPtr(), csc.innerIndexPtr(), csc.valuePtr());
|
||||
}
|
||||
|
||||
void upload_compressed(const StorageIndex* outer, const StorageIndex* inner, const Scalar* values) {
|
||||
const size_t rowptr_bytes = static_cast<size_t>(n_ + 1) * sizeof(StorageIndex);
|
||||
const size_t colidx_bytes = static_cast<size_t>(nnz_) * sizeof(StorageIndex);
|
||||
const size_t values_bytes = static_cast<size_t>(nnz_) * sizeof(Scalar);
|
||||
|
||||
d_rowPtr_ = DeviceBuffer(rowptr_bytes);
|
||||
d_colIdx_ = DeviceBuffer(colidx_bytes);
|
||||
d_values_ = DeviceBuffer(values_bytes);
|
||||
|
||||
EIGEN_CUDA_RUNTIME_CHECK(cudaMemcpyAsync(d_rowPtr_.ptr, outer, rowptr_bytes, cudaMemcpyHostToDevice, stream_));
|
||||
EIGEN_CUDA_RUNTIME_CHECK(cudaMemcpyAsync(d_colIdx_.ptr, inner, colidx_bytes, cudaMemcpyHostToDevice, stream_));
|
||||
EIGEN_CUDA_RUNTIME_CHECK(cudaMemcpyAsync(d_values_.ptr, values, values_bytes, cudaMemcpyHostToDevice, stream_));
|
||||
}
|
||||
|
||||
void create_cudss_matrix() {
|
||||
if (d_A_cudss_) (void)cudssMatrixDestroy(d_A_cudss_);
|
||||
|
||||
constexpr cudaDataType_t idx_type = cudss_index_type<StorageIndex>::value;
|
||||
constexpr cudaDataType_t val_type = cuda_data_type<Scalar>::value;
|
||||
constexpr cudssMatrixType_t mtype = Derived::cudss_matrix_type();
|
||||
constexpr cudssMatrixViewType_t mview = Derived::cudss_matrix_view();
|
||||
|
||||
EIGEN_CUDSS_CHECK(cudssMatrixCreateCsr(
|
||||
&d_A_cudss_, static_cast<int64_t>(n_), static_cast<int64_t>(n_), static_cast<int64_t>(nnz_), d_rowPtr_.ptr,
|
||||
/*rowEnd=*/nullptr, d_colIdx_.ptr, d_values_.ptr, idx_type, val_type, mtype, mview, CUDSS_BASE_ZERO));
|
||||
}
|
||||
|
||||
void apply_ordering_config() {
|
||||
cudssAlgType_t alg;
|
||||
switch (ordering_) {
|
||||
case GpuSparseOrdering::AMD:
|
||||
alg = CUDSS_ALG_DEFAULT;
|
||||
break;
|
||||
case GpuSparseOrdering::METIS:
|
||||
alg = CUDSS_ALG_2;
|
||||
break;
|
||||
case GpuSparseOrdering::RCM:
|
||||
alg = CUDSS_ALG_3;
|
||||
break;
|
||||
default:
|
||||
alg = CUDSS_ALG_DEFAULT;
|
||||
break;
|
||||
}
|
||||
EIGEN_CUDSS_CHECK(cudssConfigSet(config_, CUDSS_CONFIG_REORDERING_ALG, &alg, sizeof(alg)));
|
||||
}
|
||||
|
||||
void create_placeholder_dense() {
|
||||
if (d_x_cudss_) (void)cudssMatrixDestroy(d_x_cudss_);
|
||||
if (d_b_cudss_) (void)cudssMatrixDestroy(d_b_cudss_);
|
||||
constexpr cudaDataType_t dtype = cuda_data_type<Scalar>::value;
|
||||
EIGEN_CUDSS_CHECK(cudssMatrixCreateDn(&d_x_cudss_, static_cast<int64_t>(n_), 1, static_cast<int64_t>(n_), nullptr,
|
||||
dtype, CUDSS_LAYOUT_COL_MAJOR));
|
||||
EIGEN_CUDSS_CHECK(cudssMatrixCreateDn(&d_b_cudss_, static_cast<int64_t>(n_), 1, static_cast<int64_t>(n_), nullptr,
|
||||
dtype, CUDSS_LAYOUT_COL_MAJOR));
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace internal
|
||||
} // namespace Eigen
|
||||
|
||||
#endif // EIGEN_GPU_SPARSE_SOLVER_BASE_H
|
||||
@@ -1,8 +1,8 @@
|
||||
# Eigen GPU Module (`Eigen/GPU`)
|
||||
|
||||
GPU-accelerated dense linear algebra for Eigen users, dispatching to NVIDIA
|
||||
CUDA libraries (cuBLAS, cuSOLVER). Requires CUDA 11.4+. Header-only (link
|
||||
against CUDA runtime, cuBLAS, and cuSOLVER).
|
||||
GPU-accelerated linear algebra for Eigen users, dispatching to NVIDIA CUDA
|
||||
libraries (cuBLAS, cuSOLVER, cuFFT, cuSPARSE, cuDSS). Requires CUDA 11.4+;
|
||||
cuDSS features require CUDA 12.0+ and a separate cuDSS install. Header-only.
|
||||
|
||||
## Why this module
|
||||
|
||||
@@ -10,25 +10,31 @@ Eigen is the linear algebra foundation for a large ecosystem of C++ projects
|
||||
in robotics (ROS, Drake, MoveIt, Pinocchio), computer vision (OpenCV, COLMAP,
|
||||
Open3D), scientific computing (Ceres, Stan), and beyond. Many of these
|
||||
projects run on GPU-equipped hardware but cannot use GPUs for Eigen operations
|
||||
without dropping down to raw CUDA library APIs. Third-party projects like
|
||||
[EigenCuda](https://github.com/NLESC-JCER/EigenCuda) and
|
||||
[cholespy](https://github.com/rgl-epfl/cholespy) exist specifically to fill
|
||||
this gap, and downstream projects like
|
||||
[Ceres](https://github.com/ceres-solver/ceres-solver/issues/1151) and
|
||||
[COLMAP](https://github.com/colmap/colmap/issues/4018) have open requests for
|
||||
GPU-accelerated solvers through Eigen.
|
||||
without dropping down to raw CUDA library APIs.
|
||||
|
||||
The `Eigen/GPU` module aims to close this gap: Existing Eigen users should be
|
||||
able to move performance-critical dense linear algebra to the GPU with minimal
|
||||
code changes and without learning CUDA library APIs directly.
|
||||
GPU sparse solvers are a particularly acute gap. Sparse factorization is the
|
||||
bottleneck in SLAM, bundle adjustment, FEM, and nonlinear optimization --
|
||||
exactly the workloads where GPU acceleration matters most. Downstream projects
|
||||
like [Ceres](https://github.com/ceres-solver/ceres-solver/issues/1151) and
|
||||
[COLMAP](https://github.com/colmap/colmap/issues/4018) have open requests for
|
||||
GPU-accelerated sparse solvers, and third-party projects like
|
||||
[cholespy](https://github.com/rgl-epfl/cholespy) exist specifically because
|
||||
Eigen lacks them. The `Eigen/GPU` module provides GPU sparse Cholesky, LDL^T,
|
||||
and LU factorization via cuDSS, alongside dense solvers (cuSOLVER), matrix
|
||||
products (cuBLAS), FFT (cuFFT), and sparse matrix-vector products (cuSPARSE).
|
||||
|
||||
Existing Eigen users should be able to move performance-critical dense or
|
||||
sparse linear algebra to the GPU with minimal code changes and without
|
||||
learning CUDA library APIs directly.
|
||||
|
||||
## Design philosophy
|
||||
|
||||
**CPU and GPU coexist.** There is no global compile-time switch that replaces
|
||||
CPU implementations (unlike `EIGEN_USE_LAPACKE`). Users choose GPU solvers
|
||||
explicitly -- `GpuLLT<double>` vs `LLT<MatrixXd>` -- and both coexist in
|
||||
the same binary. This also lets users keep the factored matrix on device across
|
||||
multiple solves, something impossible with compile-time replacement.
|
||||
explicitly -- `GpuLLT<double>` vs `LLT<MatrixXd>`, `GpuSparseLLT<double>` vs
|
||||
`SimplicialLLT<SparseMatrix<double>>` -- and both coexist in the same binary.
|
||||
This also lets users keep the factored matrix on device across multiple solves,
|
||||
something impossible with compile-time replacement.
|
||||
|
||||
**Familiar syntax.** GPU operations use the same expression patterns as CPU
|
||||
Eigen. Here is a side-by-side comparison:
|
||||
@@ -38,6 +44,7 @@ Eigen. Here is a side-by-side comparison:
|
||||
#include <Eigen/Dense> #define EIGEN_USE_GPU
|
||||
#include <Eigen/GPU>
|
||||
|
||||
// Dense
|
||||
MatrixXd A = ...; auto d_A = DeviceMatrix<double>::fromHost(A);
|
||||
MatrixXd B = ...; auto d_B = DeviceMatrix<double>::fromHost(B);
|
||||
|
||||
@@ -45,11 +52,15 @@ MatrixXd C = A * B; DeviceMatrix<double> d_C = d_A * d_B;
|
||||
MatrixXd X = A.llt().solve(B); DeviceMatrix<double> d_X = d_A.llt().solve(d_B);
|
||||
|
||||
MatrixXd X = d_X.toHost();
|
||||
|
||||
// Sparse (using SpMat = SparseMatrix<double>)
|
||||
SimplicialLLT<SpMat> llt(A); GpuSparseLLT<double> llt(A);
|
||||
VectorXd x = llt.solve(b); VectorXd x = llt.solve(b);
|
||||
```
|
||||
|
||||
The GPU version reads like CPU Eigen with explicit upload/download.
|
||||
`operator*` dispatches to cuBLAS GEMM, `.llt().solve()` dispatches to
|
||||
cuSOLVER potrf + potrs. Unsupported expressions are compile errors.
|
||||
The GPU version reads like CPU Eigen with explicit upload/download for dense
|
||||
operations, and an almost identical API for sparse solvers. Unsupported
|
||||
expressions are compile errors.
|
||||
|
||||
**Explicit over implicit.** Host-device transfers, stream management, and
|
||||
library handle lifetimes are visible in the API. There are no hidden
|
||||
@@ -162,24 +173,94 @@ lu.compute(d_A);
|
||||
auto d_Y = lu.solve(d_B, GpuLU<double>::Transpose); // A^T Y = B
|
||||
|
||||
// QR solve (overdetermined least squares)
|
||||
GpuQR<double> qr(A); // host matrix input
|
||||
MatrixXd X = qr.solve(B); // Q^H * B via ormqr, then trsm on R
|
||||
GpuQR<double> qr;
|
||||
qr.compute(d_A); // factorize on device (async)
|
||||
auto d_X = qr.solve(d_B); // Q^H * B via ormqr, then trsm on R
|
||||
MatrixXd X = d_X.toHost();
|
||||
|
||||
// SVD
|
||||
GpuSVD<double> svd(A, ComputeThinU | ComputeThinV);
|
||||
VectorXd S = svd.singularValues();
|
||||
MatrixXd U = svd.matrixU();
|
||||
MatrixXd VT = svd.matrixVT();
|
||||
MatrixXd X = svd.solve(B); // pseudoinverse solve
|
||||
// SVD (results downloaded on access)
|
||||
GpuSVD<double> svd;
|
||||
svd.compute(d_A, ComputeThinU | ComputeThinV);
|
||||
VectorXd S = svd.singularValues(); // downloads to host
|
||||
MatrixXd U = svd.matrixU(); // downloads to host
|
||||
MatrixXd VT = svd.matrixVT(); // V^T (matches cuSOLVER)
|
||||
|
||||
// Self-adjoint eigenvalue decomposition
|
||||
GpuSelfAdjointEigenSolver<double> es(A);
|
||||
VectorXd eigenvals = es.eigenvalues();
|
||||
MatrixXd eigenvecs = es.eigenvectors();
|
||||
// Self-adjoint eigenvalue decomposition (results downloaded on access)
|
||||
GpuSelfAdjointEigenSolver<double> es;
|
||||
es.compute(d_A);
|
||||
VectorXd eigenvals = es.eigenvalues(); // downloads to host
|
||||
MatrixXd eigenvecs = es.eigenvectors(); // downloads to host
|
||||
```
|
||||
|
||||
The cached API keeps the factored matrix on device, avoiding redundant
|
||||
host-device transfers and re-factorizations.
|
||||
host-device transfers and re-factorizations. All solvers also accept host
|
||||
matrices directly as a convenience (e.g., `GpuLLT<double> llt(A)` or
|
||||
`qr.solve(B)`), which handles upload/download internally.
|
||||
|
||||
### Sparse direct solvers (cuDSS)
|
||||
|
||||
Requires cuDSS (separate install, CUDA 12.0+). Define `EIGEN_CUDSS` before
|
||||
including `Eigen/GPU` and link with `-lcudss`.
|
||||
|
||||
```cpp
|
||||
SparseMatrix<double> A = ...; // symmetric positive definite
|
||||
VectorXd b = ...;
|
||||
|
||||
// Sparse Cholesky -- one-liner
|
||||
GpuSparseLLT<double> llt(A);
|
||||
VectorXd x = llt.solve(b);
|
||||
|
||||
// Three-phase workflow for repeated solves with the same sparsity pattern
|
||||
GpuSparseLLT<double> llt;
|
||||
llt.analyzePattern(A); // symbolic analysis (once)
|
||||
llt.factorize(A); // numeric factorization
|
||||
VectorXd x = llt.solve(b);
|
||||
llt.factorize(A_new_values); // refactorize (reuses symbolic analysis)
|
||||
VectorXd x2 = llt.solve(b);
|
||||
|
||||
// Sparse LDL^T (symmetric indefinite)
|
||||
GpuSparseLDLT<double> ldlt(A);
|
||||
VectorXd x = ldlt.solve(b);
|
||||
|
||||
// Sparse LU (general non-symmetric)
|
||||
GpuSparseLU<double> lu(A);
|
||||
VectorXd x = lu.solve(b);
|
||||
```
|
||||
|
||||
### FFT (cuFFT)
|
||||
|
||||
```cpp
|
||||
GpuFFT<float> fft;
|
||||
|
||||
// 1D complex-to-complex
|
||||
VectorXcf X = fft.fwd(x); // forward
|
||||
VectorXcf y = fft.inv(X); // inverse (scaled by 1/n)
|
||||
|
||||
// 1D real-to-complex / complex-to-real
|
||||
VectorXcf R = fft.fwd(r); // returns n/2+1 complex (half-spectrum)
|
||||
VectorXf s = fft.invReal(R, n); // C2R inverse, caller specifies n
|
||||
|
||||
// 2D complex-to-complex
|
||||
MatrixXcf B = fft.fwd2d(A); // 2D forward
|
||||
MatrixXcf C = fft.inv2d(B); // 2D inverse (scaled by 1/(rows*cols))
|
||||
|
||||
// Plans are cached and reused across calls with the same size/type.
|
||||
```
|
||||
|
||||
### Sparse matrix-vector multiply (cuSPARSE)
|
||||
|
||||
```cpp
|
||||
SparseMatrix<double> A = ...;
|
||||
VectorXd x = ...;
|
||||
|
||||
GpuSparseContext<double> ctx;
|
||||
VectorXd y = ctx.multiply(A, x); // y = A * x
|
||||
VectorXd z = ctx.multiplyT(A, x); // z = A^T * x
|
||||
ctx.multiply(A, x, y, 2.0, 1.0); // y = 2*A*x + y
|
||||
|
||||
// Multiple RHS (SpMM)
|
||||
MatrixXd Y = ctx.multiplyMat(A, X); // Y = A * X
|
||||
```
|
||||
|
||||
### Precision control
|
||||
|
||||
@@ -219,7 +300,8 @@ skip the wait (CUDA guarantees in-order execution within a stream).
|
||||
|
||||
### Supported scalar types
|
||||
|
||||
`float`, `double`, `std::complex<float>`, `std::complex<double>`.
|
||||
`float`, `double`, `std::complex<float>`, `std::complex<double>` (unless
|
||||
noted otherwise).
|
||||
|
||||
### Expression -> library call mapping
|
||||
|
||||
@@ -241,29 +323,41 @@ skip the wait (CUDA guarantees in-order execution within a stream).
|
||||
| `C = A.selfadjointView<L>() * B` | `cublasXsymm` / `cublasXhemm` | side=L, uplo |
|
||||
| `C.selfadjointView<L>().rankUpdate(A)` | `cublasXsyrk` / `cublasXherk` | uplo, trans=N |
|
||||
|
||||
### `DeviceMatrix<Scalar>` API
|
||||
### `DeviceMatrix<Scalar>`
|
||||
|
||||
| Method | Sync? | Description |
|
||||
|--------|-------|-------------|
|
||||
| `DeviceMatrix()` | -- | Empty (0x0) |
|
||||
| `DeviceMatrix(rows, cols)` | -- | Allocate uninitialized |
|
||||
| `fromHost(matrix, stream)` | yes | Upload from Eigen matrix |
|
||||
| `fromHostAsync(ptr, rows, cols, outerStride, stream)` | no | Async upload (caller manages lifetime) |
|
||||
| `toHost(stream)` | yes | Synchronous download |
|
||||
| `toHostAsync(stream)` | no | Returns `HostTransfer` future |
|
||||
| `clone(stream)` | no | Device-to-device deep copy |
|
||||
| `resize(rows, cols)` | -- | Discard contents, reallocate |
|
||||
| `data()` | -- | Raw device pointer |
|
||||
| `rows()`, `cols()` | -- | Dimensions |
|
||||
| `sizeInBytes()` | -- | Total device allocation size in bytes |
|
||||
| `empty()` | -- | True if 0x0 |
|
||||
| `adjoint()` | -- | Adjoint view (GEMM ConjTrans) |
|
||||
| `transpose()` | -- | Transpose view (GEMM Trans) |
|
||||
| `llt()` / `llt<UpLo>()` | -- | Cholesky expression builder |
|
||||
| `lu()` | -- | LU expression builder |
|
||||
| `triangularView<UpLo>()` | -- | Triangular view (TRSM) |
|
||||
| `selfadjointView<UpLo>()` | -- | Self-adjoint view (SYMM, rankUpdate) |
|
||||
| `device(ctx)` | -- | Assignment proxy bound to context |
|
||||
Typed RAII wrapper for a dense column-major matrix in GPU device memory.
|
||||
Always dense (leading dimension = rows). A vector is a `DeviceMatrix` with
|
||||
one column.
|
||||
|
||||
```cpp
|
||||
// Construction
|
||||
DeviceMatrix<Scalar>() // Empty (0x0)
|
||||
DeviceMatrix<Scalar>(rows, cols) // Allocate uninitialized
|
||||
|
||||
// Upload / download
|
||||
static DeviceMatrix fromHost(matrix, stream=nullptr) // -> DeviceMatrix (syncs)
|
||||
static DeviceMatrix fromHostAsync(ptr, rows, cols, outerStride, s) // -> DeviceMatrix (no sync, caller manages ptr lifetime)
|
||||
PlainMatrix toHost(stream=nullptr) // -> host Matrix (syncs)
|
||||
HostTransfer toHostAsync(stream=nullptr) // -> HostTransfer future (no sync)
|
||||
DeviceMatrix clone(stream=nullptr) // -> DeviceMatrix (D2D copy, async)
|
||||
|
||||
// Dimensions and access
|
||||
Index rows()
|
||||
Index cols()
|
||||
size_t sizeInBytes()
|
||||
bool empty()
|
||||
Scalar* data() // Raw device pointer
|
||||
void resize(Index rows, Index cols) // Discard contents, reallocate
|
||||
|
||||
// Expression builders (return lightweight views, evaluated on assignment)
|
||||
AdjointView adjoint() // GEMM with ConjTrans
|
||||
TransposeView transpose() // GEMM with Trans
|
||||
LltExpr llt() / llt<UpLo>() // -> .solve(d_B) -> DeviceMatrix
|
||||
LuExpr lu() // -> .solve(d_B) -> DeviceMatrix
|
||||
TriangularView triangularView<UpLo>() // -> .solve(d_B) -> DeviceMatrix (TRSM)
|
||||
SelfAdjointView selfadjointView<UpLo>() // -> * d_B (SYMM), .rankUpdate(d_A) (SYRK)
|
||||
DeviceAssignment device(GpuContext& ctx) // Bind assignment to explicit stream
|
||||
```
|
||||
|
||||
### `GpuContext`
|
||||
|
||||
@@ -280,92 +374,190 @@ cusolverDnHandle_t cusolverHandle()
|
||||
|
||||
Non-copyable, non-movable (owns library handles).
|
||||
|
||||
### `GpuLLT<Scalar, UpLo>` API
|
||||
### `GpuLLT<Scalar, UpLo>` -- Dense Cholesky (cuSOLVER)
|
||||
|
||||
GPU dense Cholesky (LL^T) via cuSOLVER. Caches factor on device.
|
||||
Caches the Cholesky factor on device for repeated solves.
|
||||
|
||||
| Method | Sync? | Description |
|
||||
|--------|-------|-------------|
|
||||
| `GpuLLT(A)` | deferred | Construct and factorize from host matrix |
|
||||
| `compute(host_matrix)` | deferred | Upload and factorize |
|
||||
| `compute(DeviceMatrix)` | deferred | D2D copy and factorize |
|
||||
| `compute(DeviceMatrix&&)` | deferred | Move-adopt and factorize (no copy) |
|
||||
| `solve(host_matrix)` | yes | Solve, return host matrix |
|
||||
| `solve(DeviceMatrix)` | no | Solve, return `DeviceMatrix` (async) |
|
||||
| `info()` | lazy | Syncs stream on first call, returns `Success` or `NumericalIssue` |
|
||||
```cpp
|
||||
GpuLLT() // Default construct, then call compute()
|
||||
GpuLLT(const EigenBase<D>& A) // Convenience: upload + factorize
|
||||
|
||||
### `GpuLU<Scalar>` API
|
||||
GpuLLT& compute(const EigenBase<D>& A) // Upload + factorize
|
||||
GpuLLT& compute(const DeviceMatrix& d_A) // D2D copy + factorize
|
||||
GpuLLT& compute(DeviceMatrix&& d_A) // Adopt + factorize (no copy)
|
||||
|
||||
GPU dense partial-pivoting LU via cuSOLVER. Same pattern as `GpuLLT`, plus
|
||||
`TransposeMode` parameter on `solve()` (`NoTranspose`, `Transpose`,
|
||||
`ConjugateTranspose`).
|
||||
PlainMatrix solve(const MatrixBase<D>& B) // -> host Matrix (syncs)
|
||||
DeviceMatrix solve(const DeviceMatrix& d_B) // -> DeviceMatrix (async, stays on device)
|
||||
|
||||
### `GpuQR<Scalar>` API
|
||||
ComputationInfo info() // Lazy sync on first call: Success or NumericalIssue
|
||||
Index rows() / cols()
|
||||
cudaStream_t stream()
|
||||
```
|
||||
|
||||
GPU dense QR decomposition via cuSOLVER (`geqrf`). Solve uses `ormqr` (apply
|
||||
Q^H) + `trsm` (back-substitute on R) -- Q is never formed explicitly.
|
||||
### `GpuLU<Scalar>` -- Dense LU (cuSOLVER)
|
||||
|
||||
| Method | Description |
|
||||
|--------|-------------|
|
||||
| `GpuQR()` | Default construct, then call `compute()` |
|
||||
| `GpuQR(A)` | Construct and factorize from host matrix |
|
||||
| `compute(A)` | Upload + factorize |
|
||||
| `compute(DeviceMatrix)` | D2D copy + factorize |
|
||||
| `solve(host_matrix)` | Solve, return host matrix (syncs) |
|
||||
| `solve(DeviceMatrix)` | Solve, return `DeviceMatrix` (async) |
|
||||
| `info()` | Lazy sync |
|
||||
| `rows()`, `cols()`, `stream()` | Dimensions and CUDA stream |
|
||||
Same pattern as `GpuLLT`. Adds `TransposeMode` parameter on `solve()`.
|
||||
|
||||
### `GpuSVD<Scalar>` API
|
||||
```cpp
|
||||
PlainMatrix solve(const MatrixBase<D>& B, TransposeMode m = NoTranspose) // -> host Matrix
|
||||
DeviceMatrix solve(const DeviceMatrix& d_B, TransposeMode m = NoTranspose) // -> DeviceMatrix
|
||||
```
|
||||
|
||||
GPU dense SVD via cuSOLVER (`gesvd`). Supports thin, full, and values-only
|
||||
modes via Eigen's `ComputeThinU | ComputeThinV`, `ComputeFullU | ComputeFullV`,
|
||||
or `0` (values only).
|
||||
`TransposeMode`: `NoTranspose`, `Transpose`, `ConjugateTranspose`.
|
||||
|
||||
| Method | Description |
|
||||
|--------|-------------|
|
||||
| `GpuSVD()` | Default construct, then call `compute()` |
|
||||
| `GpuSVD(A, options)` | Construct and compute (options default: `ComputeThinU \| ComputeThinV`) |
|
||||
| `compute(A, options)` | Compute from host matrix |
|
||||
| `compute(DeviceMatrix, options)` | Compute from device matrix |
|
||||
| `singularValues()` | Download singular values to host |
|
||||
| `matrixU()` | Download U to host (requires `ComputeThinU` or `ComputeFullU`) |
|
||||
| `matrixVT()` | Download V^T to host (requires `ComputeThinV` or `ComputeFullV`) |
|
||||
| `solve(B)` | Pseudoinverse solve (returns host matrix) |
|
||||
| `solve(B, k)` | Truncated solve (top k singular triplets) |
|
||||
| `solve(B, lambda)` | Tikhonov regularized solve |
|
||||
| `rank(threshold)` | Count singular values above threshold |
|
||||
| `info()` | Lazy sync |
|
||||
| `rows()`, `cols()`, `stream()` | Dimensions and CUDA stream |
|
||||
### `GpuQR<Scalar>` -- Dense QR (cuSOLVER)
|
||||
|
||||
Wide matrices (m < n) are handled by internally transposing via cuBLAS `geam`.
|
||||
QR factorization via `cusolverDnXgeqrf`. Solve uses ORMQR (apply Q^H) + TRSM
|
||||
(back-substitute on R) -- Q is never formed explicitly.
|
||||
|
||||
### `GpuSelfAdjointEigenSolver<Scalar>` API
|
||||
```cpp
|
||||
GpuQR() // Default construct
|
||||
GpuQR(const EigenBase<D>& A) // Convenience: upload + factorize
|
||||
|
||||
GPU symmetric/Hermitian eigenvalue decomposition via cuSOLVER (`syevd`).
|
||||
GpuQR& compute(const EigenBase<D>& A) // Upload + factorize
|
||||
GpuQR& compute(const DeviceMatrix& d_A) // D2D copy + factorize
|
||||
|
||||
| Method | Description |
|
||||
|--------|-------------|
|
||||
| `GpuSelfAdjointEigenSolver()` | Default construct, then call `compute()` |
|
||||
| `GpuSelfAdjointEigenSolver(A, mode)` | Construct and compute (mode default: `ComputeEigenvectors`) |
|
||||
| `compute(A, mode)` | Compute from host matrix |
|
||||
| `compute(DeviceMatrix, mode)` | Compute from device matrix |
|
||||
| `eigenvalues()` | Download eigenvalues to host (ascending order) |
|
||||
| `eigenvectors()` | Download eigenvectors to host (columns) |
|
||||
| `info()` | Lazy sync |
|
||||
| `rows()`, `cols()`, `stream()` | Dimensions and CUDA stream |
|
||||
PlainMatrix solve(const MatrixBase<D>& B) // -> host Matrix (syncs)
|
||||
DeviceMatrix solve(const DeviceMatrix& d_B) // -> DeviceMatrix (async)
|
||||
|
||||
`ComputeMode`: `GpuSelfAdjointEigenSolver::EigenvaluesOnly` or
|
||||
`GpuSelfAdjointEigenSolver::ComputeEigenvectors`.
|
||||
ComputationInfo info() // Lazy sync
|
||||
Index rows() / cols()
|
||||
cudaStream_t stream()
|
||||
```
|
||||
|
||||
### `HostTransfer<Scalar>` API
|
||||
### `GpuSVD<Scalar>` -- Dense SVD (cuSOLVER)
|
||||
|
||||
Future for async device-to-host transfer.
|
||||
SVD via `cusolverDnXgesvd`. Supports `ComputeThinU | ComputeThinV`,
|
||||
`ComputeFullU | ComputeFullV`, or `0` (values only). Wide matrices (m < n)
|
||||
handled by internal transpose.
|
||||
|
||||
| Method | Description |
|
||||
|--------|-------------|
|
||||
| `get()` | Block until transfer completes, return host matrix reference. Idempotent. |
|
||||
| `ready()` | Non-blocking poll |
|
||||
```cpp
|
||||
GpuSVD() // Default construct, then call compute()
|
||||
GpuSVD(const EigenBase<D>& A, unsigned options = ComputeThinU | ComputeThinV) // Convenience
|
||||
|
||||
GpuSVD& compute(const EigenBase<D>& A, unsigned options = ComputeThinU | ComputeThinV)
|
||||
GpuSVD& compute(const DeviceMatrix& d_A, unsigned options = ComputeThinU | ComputeThinV)
|
||||
|
||||
RealVector singularValues() // -> host vector (syncs, downloads)
|
||||
PlainMatrix matrixU() // -> host Matrix (syncs, downloads)
|
||||
PlainMatrix matrixVT() // -> host Matrix (syncs, downloads V^T)
|
||||
|
||||
PlainMatrix solve(const MatrixBase<D>& B) // -> host Matrix (pseudoinverse)
|
||||
PlainMatrix solve(const MatrixBase<D>& B, Index k) // Truncated (top k triplets)
|
||||
PlainMatrix solve(const MatrixBase<D>& B, RealScalar l) // Tikhonov regularized
|
||||
|
||||
Index rank(RealScalar threshold = -1)
|
||||
ComputationInfo info() // Lazy sync
|
||||
Index rows() / cols()
|
||||
cudaStream_t stream()
|
||||
```
|
||||
|
||||
**Note:** `singularValues()`, `matrixU()`, and `matrixVT()` download to host
|
||||
on each call. Device-side accessors returning `DeviceMatrix` are planned but
|
||||
not yet implemented.
|
||||
|
||||
### `GpuSelfAdjointEigenSolver<Scalar>` -- Eigendecomposition (cuSOLVER)
|
||||
|
||||
Symmetric/Hermitian eigenvalue decomposition via `cusolverDnXsyevd`.
|
||||
`ComputeMode` enum: `EigenvaluesOnly`, `ComputeEigenvectors`.
|
||||
|
||||
```cpp
|
||||
GpuSelfAdjointEigenSolver() // Default construct, then call compute()
|
||||
GpuSelfAdjointEigenSolver(const EigenBase<D>& A, ComputeMode mode = ComputeEigenvectors) // Convenience
|
||||
|
||||
GpuSelfAdjointEigenSolver& compute(const EigenBase<D>& A, ComputeMode mode = ComputeEigenvectors)
|
||||
GpuSelfAdjointEigenSolver& compute(const DeviceMatrix& d_A, ComputeMode mode = ComputeEigenvectors)
|
||||
|
||||
RealVector eigenvalues() // -> host vector (syncs, downloads, ascending order)
|
||||
PlainMatrix eigenvectors() // -> host Matrix (syncs, downloads, columns)
|
||||
|
||||
ComputationInfo info() // Lazy sync
|
||||
Index rows() / cols()
|
||||
cudaStream_t stream()
|
||||
```
|
||||
|
||||
**Note:** `eigenvalues()` and `eigenvectors()` download to host on each call.
|
||||
Device-side accessors returning `DeviceMatrix` are planned but not yet
|
||||
implemented.
|
||||
|
||||
### `HostTransfer<Scalar>`
|
||||
|
||||
Future for async device-to-host transfer. Returned by
|
||||
`DeviceMatrix::toHostAsync()`.
|
||||
|
||||
```cpp
|
||||
PlainMatrix& get() // Block until complete, return host Matrix ref. Idempotent.
|
||||
bool ready() // Non-blocking poll
|
||||
```
|
||||
|
||||
### `GpuSparseLLT<Scalar, UpLo>` -- Sparse Cholesky (cuDSS)
|
||||
|
||||
Requires cuDSS (CUDA 12.0+, `#define EIGEN_CUDSS`). Three-phase workflow
|
||||
with symbolic reuse. Accepts `SparseMatrix<Scalar, ColMajor, int>` (CSC).
|
||||
|
||||
```cpp
|
||||
GpuSparseLLT() // Default construct
|
||||
GpuSparseLLT(const SparseMatrixBase<D>& A) // Analyze + factorize
|
||||
|
||||
GpuSparseLLT& analyzePattern(const SparseMatrixBase<D>& A) // Symbolic analysis (reusable)
|
||||
GpuSparseLLT& factorize(const SparseMatrixBase<D>& A) // Numeric factorization
|
||||
GpuSparseLLT& compute(const SparseMatrixBase<D>& A) // analyzePattern + factorize
|
||||
void setOrdering(GpuSparseOrdering ord) // AMD (default), METIS, or RCM
|
||||
|
||||
DenseMatrix solve(const MatrixBase<D>& B) // -> host Matrix (syncs)
|
||||
|
||||
ComputationInfo info() // Lazy sync
|
||||
Index rows() / cols()
|
||||
cudaStream_t stream()
|
||||
```
|
||||
|
||||
### `GpuSparseLDLT<Scalar, UpLo>` -- Sparse LDL^T (cuDSS)
|
||||
|
||||
Symmetric indefinite. Same API as `GpuSparseLLT`.
|
||||
|
||||
### `GpuSparseLU<Scalar>` -- Sparse LU (cuDSS)
|
||||
|
||||
General non-symmetric. Same API as `GpuSparseLLT` (without `UpLo`).
|
||||
|
||||
### `GpuFFT<Scalar>` -- FFT (cuFFT)
|
||||
|
||||
Plans cached by (size, type) and reused. Inverse transforms scaled so
|
||||
`inv(fwd(x)) == x`. Supported scalars: `float`, `double`.
|
||||
|
||||
```cpp
|
||||
// 1D transforms (host vectors in and out)
|
||||
ComplexVector fwd(const MatrixBase<D>& x) // C2C forward (complex input)
|
||||
ComplexVector fwd(const MatrixBase<D>& x) // R2C forward (real input, returns n/2+1)
|
||||
ComplexVector inv(const MatrixBase<D>& X) // C2C inverse, scaled by 1/n
|
||||
RealVector invReal(const MatrixBase<D>& X, Index n) // C2R inverse, scaled by 1/n
|
||||
|
||||
// 2D transforms (host matrices in and out)
|
||||
ComplexMatrix fwd2d(const MatrixBase<D>& A) // 2D C2C forward
|
||||
ComplexMatrix inv2d(const MatrixBase<D>& A) // 2D C2C inverse, scaled by 1/(rows*cols)
|
||||
|
||||
cudaStream_t stream()
|
||||
```
|
||||
|
||||
All FFT methods accept host data and return host data. Upload/download is
|
||||
handled internally. The C2C and R2C overloads of `fwd()` are distinguished by
|
||||
the input scalar type (complex vs real).
|
||||
|
||||
### `GpuSparseContext<Scalar>` -- SpMV/SpMM (cuSPARSE)
|
||||
|
||||
Accepts `SparseMatrix<Scalar, ColMajor>`. All methods accept host data and
|
||||
return host data.
|
||||
|
||||
```cpp
|
||||
GpuSparseContext() // Creates own stream + cuSPARSE handle
|
||||
|
||||
DenseVector multiply(A, x) // y = A * x
|
||||
void multiply(A, x, y, alpha=1, beta=0, // y = alpha*op(A)*x + beta*y
|
||||
op=CUSPARSE_OPERATION_NON_TRANSPOSE)
|
||||
DenseVector multiplyT(A, x) // y = A^T * x
|
||||
DenseMatrix multiplyMat(A, X) // Y = A * X (SpMM)
|
||||
|
||||
cudaStream_t stream()
|
||||
```
|
||||
|
||||
### Aliasing
|
||||
|
||||
@@ -393,6 +585,15 @@ The caller must ensure operands don't alias the destination for GEMM and TRSM
|
||||
| `GpuQR.h` | `CuSolverSupport.h`, `CuBlasSupport.h` | Dense QR decomposition |
|
||||
| `GpuSVD.h` | `CuSolverSupport.h`, `CuBlasSupport.h` | Dense SVD decomposition |
|
||||
| `GpuEigenSolver.h` | `CuSolverSupport.h` | Self-adjoint eigenvalue decomposition |
|
||||
| `CuFftSupport.h` | `GpuSupport.h`, `<cufft.h>` | cuFFT error macro, type-dispatch wrappers |
|
||||
| `GpuFFT.h` | `CuFftSupport.h`, `CuBlasSupport.h` | 1D/2D FFT with plan caching |
|
||||
| `CuSparseSupport.h` | `GpuSupport.h`, `<cusparse.h>` | cuSPARSE error macro |
|
||||
| `GpuSparseContext.h` | `CuSparseSupport.h` | SpMV/SpMM via cuSPARSE |
|
||||
| `CuDssSupport.h` | `GpuSupport.h`, `<cudss.h>` | cuDSS error macro, type traits (optional) |
|
||||
| `GpuSparseSolverBase.h` | `CuDssSupport.h` | CRTP base for sparse solvers (optional) |
|
||||
| `GpuSparseLLT.h` | `GpuSparseSolverBase.h` | Sparse Cholesky via cuDSS (optional) |
|
||||
| `GpuSparseLDLT.h` | `GpuSparseSolverBase.h` | Sparse LDL^T via cuDSS (optional) |
|
||||
| `GpuSparseLU.h` | `GpuSparseSolverBase.h` | Sparse LU via cuDSS (optional) |
|
||||
|
||||
## Building and testing
|
||||
|
||||
@@ -404,6 +605,32 @@ cmake -G Ninja -B build -S . \
|
||||
-DEIGEN_TEST_CUSOLVER=ON
|
||||
|
||||
cmake --build build --target gpu_cublas gpu_cusolver_llt gpu_cusolver_lu \
|
||||
gpu_cusolver_qr gpu_cusolver_svd gpu_cusolver_eigen gpu_device_matrix
|
||||
ctest --test-dir build -R "gpu_cublas|gpu_cusolver|gpu_device" --output-on-failure
|
||||
gpu_cusolver_qr gpu_cusolver_svd gpu_cusolver_eigen \
|
||||
gpu_device_matrix gpu_cufft gpu_cusparse_spmv
|
||||
ctest --test-dir build -R "gpu_" --output-on-failure
|
||||
|
||||
# Sparse solvers (cuDSS -- separate install required)
|
||||
cmake -G Ninja -B build -S . \
|
||||
-DEIGEN_TEST_CUDA=ON \
|
||||
-DEIGEN_CUDA_COMPUTE_ARCH="70" \
|
||||
-DEIGEN_TEST_CUDSS=ON
|
||||
|
||||
cmake --build build --target gpu_cudss_llt gpu_cudss_ldlt gpu_cudss_lu
|
||||
ctest --test-dir build -R gpu_cudss --output-on-failure
|
||||
```
|
||||
|
||||
## Future work
|
||||
|
||||
- **Device-side accessors for decomposition results.** `GpuSVD`,
|
||||
`GpuSelfAdjointEigenSolver`, and `GpuQR` currently download decomposition
|
||||
results to host on access (e.g., `svd.matrixU()` returns a host `MatrixXd`).
|
||||
Device-side accessors returning `DeviceMatrix` views of the internal buffers
|
||||
would allow chaining GPU operations (e.g., `svd.deviceU() * d_A`) without
|
||||
round-tripping through host memory.
|
||||
- **Device-resident sparse matrix-vector products.** `GpuSparseContext`
|
||||
currently operates on host vectors and matrices, uploading and downloading
|
||||
on each call. The key missing piece is a `DeviceSparseView` that holds a
|
||||
sparse matrix on device and supports operator syntax (`d_y = d_A * d_x`)
|
||||
with `DeviceMatrix` operands -- keeping the entire SpMV/SpMM pipeline on
|
||||
device. This is essential for iterative solvers and any workflow that chains
|
||||
sparse and dense operations without returning to the host.
|
||||
|
||||
Reference in New Issue
Block a user