devtools/eigen
1
0
Fork 0
mirror of https://gitlab.com/libeigen/eigen.git synced 2026-04-10 11:34:33 +08:00
Code Issues Packages Projects Releases Wiki Activity

Apply clang-format

Browse Source
This commit is contained in:
Tobias Wood
2023-11-29 11:12:48 +00:00
parent 9ea520fc45
commit f38e16c193
534 changed files with 103368 additions and 116934 deletions
Download Patch File Download Diff File Expand all files Collapse all files

414
unsupported/Eigen/src/FFT/ei_fftw_impl.h
View File

@@ -1,5 +1,5 @@
// This file is part of Eigen, a lightweight C++ template library
// for linear algebra.
// for linear algebra.
//
// Copyright (C) 2009 Mark Borgerding mark a borgerding net
//
@@ -12,255 +12,205 @@
#include <memory>
namespace Eigen {
namespace Eigen {
namespace internal {
// FFTW uses non-const arguments
// so we must use ugly const_cast calls for all the args it uses
//
// This should be safe as long as
// 1. we use FFTW_ESTIMATE for all our planning
// see the FFTW docs section 4.3.2 "Planner Flags"
// 2. fftw_complex is compatible with std::complex
// This assumes std::complex<T> layout is array of size 2 with real,imag
template <typename T>
inline
T * fftw_cast(const T* p)
{
return const_cast<T*>( p);
// FFTW uses non-const arguments
// so we must use ugly const_cast calls for all the args it uses
//
// This should be safe as long as
// 1. we use FFTW_ESTIMATE for all our planning
// see the FFTW docs section 4.3.2 "Planner Flags"
// 2. fftw_complex is compatible with std::complex
// This assumes std::complex<T> layout is array of size 2 with real,imag
template <typename T>
inline T *fftw_cast(const T *p) {
return const_cast<T *>(p);
}
inline fftw_complex *fftw_cast(const std::complex<double> *p) {
return const_cast<fftw_complex *>(reinterpret_cast<const fftw_complex *>(p));
}
inline fftwf_complex *fftw_cast(const std::complex<float> *p) {
return const_cast<fftwf_complex *>(reinterpret_cast<const fftwf_complex *>(p));
}
inline fftwl_complex *fftw_cast(const std::complex<long double> *p) {
return const_cast<fftwl_complex *>(reinterpret_cast<const fftwl_complex *>(p));
}
template <typename T>
struct fftw_plan {};
template <>
struct fftw_plan<float> {
typedef float scalar_type;
typedef fftwf_complex complex_type;
std::shared_ptr<fftwf_plan_s> m_plan;
fftw_plan() = default;
void set_plan(fftwf_plan p) { m_plan.reset(p, fftwf_destroy_plan); }
inline void fwd(complex_type *dst, complex_type *src, int nfft) {
if (m_plan == NULL) set_plan(fftwf_plan_dft_1d(nfft, src, dst, FFTW_FORWARD, FFTW_ESTIMATE | FFTW_PRESERVE_INPUT));
fftwf_execute_dft(m_plan.get(), src, dst);
}
inline void inv(complex_type *dst, complex_type *src, int nfft) {
if (m_plan == NULL) set_plan(fftwf_plan_dft_1d(nfft, src, dst, FFTW_BACKWARD, FFTW_ESTIMATE | FFTW_PRESERVE_INPUT));
fftwf_execute_dft(m_plan.get(), src, dst);
}
inline void fwd(complex_type *dst, scalar_type *src, int nfft) {
if (m_plan == NULL) set_plan(fftwf_plan_dft_r2c_1d(nfft, src, dst, FFTW_ESTIMATE | FFTW_PRESERVE_INPUT));
fftwf_execute_dft_r2c(m_plan.get(), src, dst);
}
inline void inv(scalar_type *dst, complex_type *src, int nfft) {
if (m_plan == NULL) set_plan(fftwf_plan_dft_c2r_1d(nfft, src, dst, FFTW_ESTIMATE | FFTW_PRESERVE_INPUT));
fftwf_execute_dft_c2r(m_plan.get(), src, dst);
}
inline
fftw_complex * fftw_cast( const std::complex<double> * p)
{
return const_cast<fftw_complex*>( reinterpret_cast<const fftw_complex*>(p) );
inline void fwd2(complex_type *dst, complex_type *src, int n0, int n1) {
if (m_plan == NULL)
set_plan(fftwf_plan_dft_2d(n0, n1, src, dst, FFTW_FORWARD, FFTW_ESTIMATE | FFTW_PRESERVE_INPUT));
fftwf_execute_dft(m_plan.get(), src, dst);
}
inline void inv2(complex_type *dst, complex_type *src, int n0, int n1) {
if (m_plan == NULL)
set_plan(fftwf_plan_dft_2d(n0, n1, src, dst, FFTW_BACKWARD, FFTW_ESTIMATE | FFTW_PRESERVE_INPUT));
fftwf_execute_dft(m_plan.get(), src, dst);
}
};
template <>
struct fftw_plan<double> {
typedef double scalar_type;
typedef fftw_complex complex_type;
std::shared_ptr<fftw_plan_s> m_plan;
fftw_plan() = default;
void set_plan(::fftw_plan p) { m_plan.reset(p, fftw_destroy_plan); }
inline void fwd(complex_type *dst, complex_type *src, int nfft) {
if (m_plan == NULL) set_plan(fftw_plan_dft_1d(nfft, src, dst, FFTW_FORWARD, FFTW_ESTIMATE | FFTW_PRESERVE_INPUT));
fftw_execute_dft(m_plan.get(), src, dst);
}
inline void inv(complex_type *dst, complex_type *src, int nfft) {
if (m_plan == NULL) set_plan(fftw_plan_dft_1d(nfft, src, dst, FFTW_BACKWARD, FFTW_ESTIMATE | FFTW_PRESERVE_INPUT));
fftw_execute_dft(m_plan.get(), src, dst);
}
inline void fwd(complex_type *dst, scalar_type *src, int nfft) {
if (m_plan == NULL) set_plan(fftw_plan_dft_r2c_1d(nfft, src, dst, FFTW_ESTIMATE | FFTW_PRESERVE_INPUT));
fftw_execute_dft_r2c(m_plan.get(), src, dst);
}
inline void inv(scalar_type *dst, complex_type *src, int nfft) {
if (m_plan == NULL) set_plan(fftw_plan_dft_c2r_1d(nfft, src, dst, FFTW_ESTIMATE | FFTW_PRESERVE_INPUT));
fftw_execute_dft_c2r(m_plan.get(), src, dst);
}
inline void fwd2(complex_type *dst, complex_type *src, int n0, int n1) {
if (m_plan == NULL) set_plan(fftw_plan_dft_2d(n0, n1, src, dst, FFTW_FORWARD, FFTW_ESTIMATE | FFTW_PRESERVE_INPUT));
fftw_execute_dft(m_plan.get(), src, dst);
}
inline void inv2(complex_type *dst, complex_type *src, int n0, int n1) {
if (m_plan == NULL)
set_plan(fftw_plan_dft_2d(n0, n1, src, dst, FFTW_BACKWARD, FFTW_ESTIMATE | FFTW_PRESERVE_INPUT));
fftw_execute_dft(m_plan.get(), src, dst);
}
};
template <>
struct fftw_plan<long double> {
typedef long double scalar_type;
typedef fftwl_complex complex_type;
std::shared_ptr<fftwl_plan_s> m_plan;
fftw_plan() = default;
void set_plan(fftwl_plan p) { m_plan.reset(p, fftwl_destroy_plan); }
inline void fwd(complex_type *dst, complex_type *src, int nfft) {
if (m_plan == NULL) set_plan(fftwl_plan_dft_1d(nfft, src, dst, FFTW_FORWARD, FFTW_ESTIMATE | FFTW_PRESERVE_INPUT));
fftwl_execute_dft(m_plan.get(), src, dst);
}
inline void inv(complex_type *dst, complex_type *src, int nfft) {
if (m_plan == NULL) set_plan(fftwl_plan_dft_1d(nfft, src, dst, FFTW_BACKWARD, FFTW_ESTIMATE | FFTW_PRESERVE_INPUT));
fftwl_execute_dft(m_plan.get(), src, dst);
}
inline void fwd(complex_type *dst, scalar_type *src, int nfft) {
if (m_plan == NULL) set_plan(fftwl_plan_dft_r2c_1d(nfft, src, dst, FFTW_ESTIMATE | FFTW_PRESERVE_INPUT));
fftwl_execute_dft_r2c(m_plan.get(), src, dst);
}
inline void inv(scalar_type *dst, complex_type *src, int nfft) {
if (m_plan == NULL) set_plan(fftwl_plan_dft_c2r_1d(nfft, src, dst, FFTW_ESTIMATE | FFTW_PRESERVE_INPUT));
fftwl_execute_dft_c2r(m_plan.get(), src, dst);
}
inline void fwd2(complex_type *dst, complex_type *src, int n0, int n1) {
if (m_plan == NULL)
set_plan(fftwl_plan_dft_2d(n0, n1, src, dst, FFTW_FORWARD, FFTW_ESTIMATE | FFTW_PRESERVE_INPUT));
fftwl_execute_dft(m_plan.get(), src, dst);
}
inline void inv2(complex_type *dst, complex_type *src, int n0, int n1) {
if (m_plan == NULL)
set_plan(fftwl_plan_dft_2d(n0, n1, src, dst, FFTW_BACKWARD, FFTW_ESTIMATE | FFTW_PRESERVE_INPUT));
fftwl_execute_dft(m_plan.get(), src, dst);
}
};
template <typename Scalar_>
struct fftw_impl {
typedef Scalar_ Scalar;
typedef std::complex<Scalar> Complex;
inline void clear() { m_plans.clear(); }
// complex-to-complex forward FFT
inline void fwd(Complex *dst, const Complex *src, int nfft) {
get_plan(nfft, false, dst, src).fwd(fftw_cast(dst), fftw_cast(src), nfft);
}
inline
fftwf_complex * fftw_cast( const std::complex<float> * p)
{
return const_cast<fftwf_complex*>( reinterpret_cast<const fftwf_complex*>(p) );
// real-to-complex forward FFT
inline void fwd(Complex *dst, const Scalar *src, int nfft) {
get_plan(nfft, false, dst, src).fwd(fftw_cast(dst), fftw_cast(src), nfft);
}
inline
fftwl_complex * fftw_cast( const std::complex<long double> * p)
{
return const_cast<fftwl_complex*>( reinterpret_cast<const fftwl_complex*>(p) );
// 2-d complex-to-complex
inline void fwd2(Complex *dst, const Complex *src, int n0, int n1) {
get_plan(n0, n1, false, dst, src).fwd2(fftw_cast(dst), fftw_cast(src), n0, n1);
}
template <typename T>
struct fftw_plan {};
// inverse complex-to-complex
inline void inv(Complex *dst, const Complex *src, int nfft) {
get_plan(nfft, true, dst, src).inv(fftw_cast(dst), fftw_cast(src), nfft);
}
template <>
struct fftw_plan<float>
{
typedef float scalar_type;
typedef fftwf_complex complex_type;
std::shared_ptr<fftwf_plan_s> m_plan;
fftw_plan() = default;
// half-complex to scalar
inline void inv(Scalar *dst, const Complex *src, int nfft) {
get_plan(nfft, true, dst, src).inv(fftw_cast(dst), fftw_cast(src), nfft);
}
void set_plan(fftwf_plan p) { m_plan.reset(p, fftwf_destroy_plan); }
inline
void fwd(complex_type * dst,complex_type * src,int nfft) {
if (m_plan==NULL) set_plan(fftwf_plan_dft_1d(nfft,src,dst, FFTW_FORWARD, FFTW_ESTIMATE|FFTW_PRESERVE_INPUT));
fftwf_execute_dft( m_plan.get(), src,dst);
}
inline
void inv(complex_type * dst,complex_type * src,int nfft) {
if (m_plan==NULL) set_plan(fftwf_plan_dft_1d(nfft,src,dst, FFTW_BACKWARD , FFTW_ESTIMATE|FFTW_PRESERVE_INPUT));
fftwf_execute_dft( m_plan.get(), src,dst);
}
inline
void fwd(complex_type * dst,scalar_type * src,int nfft) {
if (m_plan==NULL) set_plan(fftwf_plan_dft_r2c_1d(nfft,src,dst,FFTW_ESTIMATE|FFTW_PRESERVE_INPUT));
fftwf_execute_dft_r2c( m_plan.get(),src,dst);
}
inline
void inv(scalar_type * dst,complex_type * src,int nfft) {
if (m_plan==NULL)
set_plan(fftwf_plan_dft_c2r_1d(nfft,src,dst,FFTW_ESTIMATE|FFTW_PRESERVE_INPUT));
fftwf_execute_dft_c2r( m_plan.get(), src,dst);
}
// 2-d complex-to-complex
inline void inv2(Complex *dst, const Complex *src, int n0, int n1) {
get_plan(n0, n1, true, dst, src).inv2(fftw_cast(dst), fftw_cast(src), n0, n1);
}
inline
void fwd2( complex_type * dst,complex_type * src,int n0,int n1) {
if (m_plan==NULL) set_plan(fftwf_plan_dft_2d(n0,n1,src,dst,FFTW_FORWARD,FFTW_ESTIMATE|FFTW_PRESERVE_INPUT));
fftwf_execute_dft( m_plan.get(), src,dst);
}
inline
void inv2( complex_type * dst,complex_type * src,int n0,int n1) {
if (m_plan==NULL) set_plan(fftwf_plan_dft_2d(n0,n1,src,dst,FFTW_BACKWARD,FFTW_ESTIMATE|FFTW_PRESERVE_INPUT));
fftwf_execute_dft( m_plan.get(), src,dst);
}
protected:
typedef fftw_plan<Scalar> PlanData;
};
template <>
struct fftw_plan<double>
{
typedef double scalar_type;
typedef fftw_complex complex_type;
std::shared_ptr<fftw_plan_s> m_plan;
fftw_plan() = default;
typedef Eigen::numext::int64_t int64_t;
void set_plan(::fftw_plan p) { m_plan.reset(p, fftw_destroy_plan); }
inline
void fwd(complex_type * dst,complex_type * src,int nfft) {
if (m_plan==NULL) set_plan(fftw_plan_dft_1d(nfft,src,dst, FFTW_FORWARD, FFTW_ESTIMATE|FFTW_PRESERVE_INPUT));
fftw_execute_dft( m_plan.get(), src,dst);
}
inline
void inv(complex_type * dst,complex_type * src,int nfft) {
if (m_plan==NULL) set_plan(fftw_plan_dft_1d(nfft,src,dst, FFTW_BACKWARD , FFTW_ESTIMATE|FFTW_PRESERVE_INPUT));
fftw_execute_dft( m_plan.get(), src,dst);
}
inline
void fwd(complex_type * dst,scalar_type * src,int nfft) {
if (m_plan==NULL) set_plan(fftw_plan_dft_r2c_1d(nfft,src,dst,FFTW_ESTIMATE|FFTW_PRESERVE_INPUT));
fftw_execute_dft_r2c( m_plan.get(),src,dst);
}
inline
void inv(scalar_type * dst,complex_type * src,int nfft) {
if (m_plan==NULL)
set_plan(fftw_plan_dft_c2r_1d(nfft,src,dst,FFTW_ESTIMATE|FFTW_PRESERVE_INPUT));
fftw_execute_dft_c2r( m_plan.get(), src,dst);
}
inline
void fwd2( complex_type * dst,complex_type * src,int n0,int n1) {
if (m_plan==NULL) set_plan(fftw_plan_dft_2d(n0,n1,src,dst,FFTW_FORWARD,FFTW_ESTIMATE|FFTW_PRESERVE_INPUT));
fftw_execute_dft( m_plan.get(), src,dst);
}
inline
void inv2( complex_type * dst,complex_type * src,int n0,int n1) {
if (m_plan==NULL) set_plan(fftw_plan_dft_2d(n0,n1,src,dst,FFTW_BACKWARD,FFTW_ESTIMATE|FFTW_PRESERVE_INPUT));
fftw_execute_dft( m_plan.get(), src,dst);
}
};
template <>
struct fftw_plan<long double>
{
typedef long double scalar_type;
typedef fftwl_complex complex_type;
std::shared_ptr<fftwl_plan_s> m_plan;
fftw_plan() = default;
typedef std::map<int64_t, PlanData> PlanMap;
void set_plan(fftwl_plan p) { m_plan.reset(p, fftwl_destroy_plan); }
inline
void fwd(complex_type * dst,complex_type * src,int nfft) {
if (m_plan==NULL) set_plan(fftwl_plan_dft_1d(nfft,src,dst, FFTW_FORWARD, FFTW_ESTIMATE|FFTW_PRESERVE_INPUT));
fftwl_execute_dft( m_plan.get(), src,dst);
}
inline
void inv(complex_type * dst,complex_type * src,int nfft) {
if (m_plan==NULL) set_plan(fftwl_plan_dft_1d(nfft,src,dst, FFTW_BACKWARD , FFTW_ESTIMATE|FFTW_PRESERVE_INPUT));
fftwl_execute_dft( m_plan.get(), src,dst);
}
inline
void fwd(complex_type * dst,scalar_type * src,int nfft) {
if (m_plan==NULL) set_plan(fftwl_plan_dft_r2c_1d(nfft,src,dst,FFTW_ESTIMATE|FFTW_PRESERVE_INPUT));
fftwl_execute_dft_r2c( m_plan.get(),src,dst);
}
inline
void inv(scalar_type * dst,complex_type * src,int nfft) {
if (m_plan==NULL)
set_plan(fftwl_plan_dft_c2r_1d(nfft,src,dst,FFTW_ESTIMATE|FFTW_PRESERVE_INPUT));
fftwl_execute_dft_c2r( m_plan.get(), src,dst);
}
inline
void fwd2( complex_type * dst,complex_type * src,int n0,int n1) {
if (m_plan==NULL) set_plan(fftwl_plan_dft_2d(n0,n1,src,dst,FFTW_FORWARD,FFTW_ESTIMATE|FFTW_PRESERVE_INPUT));
fftwl_execute_dft( m_plan.get(), src,dst);
}
inline
void inv2( complex_type * dst,complex_type * src,int n0,int n1) {
if (m_plan==NULL) set_plan(fftwl_plan_dft_2d(n0,n1,src,dst,FFTW_BACKWARD,FFTW_ESTIMATE|FFTW_PRESERVE_INPUT));
fftwl_execute_dft( m_plan.get(), src,dst);
}
};
PlanMap m_plans;
template <typename Scalar_>
struct fftw_impl
{
typedef Scalar_ Scalar;
typedef std::complex<Scalar> Complex;
inline PlanData &get_plan(int nfft, bool inverse, void *dst, const void *src) {
bool inplace = (dst == src);
bool aligned = ((reinterpret_cast<size_t>(src) & 15) | (reinterpret_cast<size_t>(dst) & 15)) == 0;
int64_t key = ((nfft << 3) | (inverse << 2) | (inplace << 1) | aligned) << 1;
return m_plans[key];
}
inline
void clear()
{
m_plans.clear();
}
inline PlanData &get_plan(int n0, int n1, bool inverse, void *dst, const void *src) {
bool inplace = (dst == src);
bool aligned = ((reinterpret_cast<size_t>(src) & 15) | (reinterpret_cast<size_t>(dst) & 15)) == 0;
int64_t key = (((((int64_t)n0) << 30) | (n1 << 3) | (inverse << 2) | (inplace << 1) | aligned) << 1) + 1;
return m_plans[key];
}
};
// complex-to-complex forward FFT
inline
void fwd( Complex * dst,const Complex *src,int nfft)
{
get_plan(nfft,false,dst,src).fwd(fftw_cast(dst), fftw_cast(src),nfft );
}
} // end namespace internal
// real-to-complex forward FFT
inline
void fwd( Complex * dst,const Scalar * src,int nfft)
{
get_plan(nfft,false,dst,src).fwd(fftw_cast(dst), fftw_cast(src) ,nfft);
}
// 2-d complex-to-complex
inline
void fwd2(Complex * dst, const Complex * src, int n0,int n1)
{
get_plan(n0,n1,false,dst,src).fwd2(fftw_cast(dst), fftw_cast(src) ,n0,n1);
}
// inverse complex-to-complex
inline
void inv(Complex * dst,const Complex *src,int nfft)
{
get_plan(nfft,true,dst,src).inv(fftw_cast(dst), fftw_cast(src),nfft );
}
// half-complex to scalar
inline
void inv( Scalar * dst,const Complex * src,int nfft)
{
get_plan(nfft,true,dst,src).inv(fftw_cast(dst), fftw_cast(src),nfft );
}
// 2-d complex-to-complex
inline
void inv2(Complex * dst, const Complex * src, int n0,int n1)
{
get_plan(n0,n1,true,dst,src).inv2(fftw_cast(dst), fftw_cast(src) ,n0,n1);
}
protected:
typedef fftw_plan<Scalar> PlanData;
typedef Eigen::numext::int64_t int64_t;
typedef std::map<int64_t,PlanData> PlanMap;
PlanMap m_plans;
inline
PlanData & get_plan(int nfft,bool inverse,void * dst,const void * src)
{
bool inplace = (dst==src);
bool aligned = ( (reinterpret_cast<size_t>(src)&15) | (reinterpret_cast<size_t>(dst)&15) ) == 0;
int64_t key = ( (nfft<<3 ) | (inverse<<2) | (inplace<<1) | aligned ) << 1;
return m_plans[key];
}
inline
PlanData & get_plan(int n0,int n1,bool inverse,void * dst,const void * src)
{
bool inplace = (dst==src);
bool aligned = ( (reinterpret_cast<size_t>(src)&15) | (reinterpret_cast<size_t>(dst)&15) ) == 0;
int64_t key = ( ( (((int64_t)n0) << 30)|(n1<<3 ) | (inverse<<2) | (inplace<<1) | aligned ) << 1 ) + 1;
return m_plans[key];
}
};
} // end namespace internal
} // end namespace Eigen
} // end namespace Eigen

107
unsupported/Eigen/src/FFT/ei_imklfft_impl.h
View File

@@ -17,9 +17,9 @@ namespace Eigen {
namespace internal {
namespace imklfft {
#define RUN_OR_ASSERT(EXPR, ERROR_MSG) \
{ \
MKL_LONG status = (EXPR); \
#define RUN_OR_ASSERT(EXPR, ERROR_MSG) \
{ \
MKL_LONG status = (EXPR); \
eigen_assert(status == DFTI_NO_ERROR && (ERROR_MSG)); \
};
@@ -40,35 +40,26 @@ inline MKL_Complex8* complex_cast(const std::complex<float>* p) {
* Array of type MKL_LONG otherwise. Lengths of each dimension for a
* multi-dimensional transform.
*/
inline void configure_descriptor(std::shared_ptr<DFTI_DESCRIPTOR>& handl,
enum DFTI_CONFIG_VALUE precision,
enum DFTI_CONFIG_VALUE forward_domain,
MKL_LONG dimension, MKL_LONG* sizes) {
eigen_assert(dimension == 1 ||
dimension == 2 &&
"Transformation dimension must be less than 3.");
inline void configure_descriptor(std::shared_ptr<DFTI_DESCRIPTOR>& handl, enum DFTI_CONFIG_VALUE precision,
enum DFTI_CONFIG_VALUE forward_domain, MKL_LONG dimension, MKL_LONG* sizes) {
eigen_assert(dimension == 1 || dimension == 2 && "Transformation dimension must be less than 3.");
DFTI_DESCRIPTOR_HANDLE res = nullptr;
if (dimension == 1) {
RUN_OR_ASSERT(DftiCreateDescriptor(&res, precision, forward_domain,
dimension, *sizes),
RUN_OR_ASSERT(DftiCreateDescriptor(&res, precision, forward_domain, dimension, *sizes),
"DftiCreateDescriptor failed.")
handl.reset(res, [](DFTI_DESCRIPTOR_HANDLE handle) { DftiFreeDescriptor(&handle); });
if (forward_domain == DFTI_REAL) {
// Set CCE storage
RUN_OR_ASSERT(DftiSetValue(handl.get(), DFTI_CONJUGATE_EVEN_STORAGE,
DFTI_COMPLEX_COMPLEX),
RUN_OR_ASSERT(DftiSetValue(handl.get(), DFTI_CONJUGATE_EVEN_STORAGE, DFTI_COMPLEX_COMPLEX),
"DftiSetValue failed.")
}
} else {
RUN_OR_ASSERT(
DftiCreateDescriptor(&res, precision, DFTI_COMPLEX, dimension, sizes),
"DftiCreateDescriptor failed.")
RUN_OR_ASSERT(DftiCreateDescriptor(&res, precision, DFTI_COMPLEX, dimension, sizes), "DftiCreateDescriptor failed.")
handl.reset(res, [](DFTI_DESCRIPTOR_HANDLE handle) { DftiFreeDescriptor(&handle); });
}
RUN_OR_ASSERT(DftiSetValue(handl.get(), DFTI_PLACEMENT, DFTI_NOT_INPLACE),
"DftiSetValue failed.")
RUN_OR_ASSERT(DftiSetValue(handl.get(), DFTI_PLACEMENT, DFTI_NOT_INPLACE), "DftiSetValue failed.")
RUN_OR_ASSERT(DftiCommitDescriptor(handl.get()), "DftiCommitDescriptor failed.")
}
@@ -90,32 +81,28 @@ struct plan<float> {
if (m_plan == 0) {
configure_descriptor(m_plan, precision, DFTI_COMPLEX, 1, &nfft);
}
RUN_OR_ASSERT(DftiComputeForward(m_plan.get(), src, dst),
"DftiComputeForward failed.")
RUN_OR_ASSERT(DftiComputeForward(m_plan.get(), src, dst), "DftiComputeForward failed.")
}
inline void inverse(complex_type* dst, complex_type* src, MKL_LONG nfft) {
if (m_plan == 0) {
configure_descriptor(m_plan, precision, DFTI_COMPLEX, 1, &nfft);
}
RUN_OR_ASSERT(DftiComputeBackward(m_plan.get(), src, dst),
"DftiComputeBackward failed.")
RUN_OR_ASSERT(DftiComputeBackward(m_plan.get(), src, dst), "DftiComputeBackward failed.")
}
inline void forward(complex_type* dst, scalar_type* src, MKL_LONG nfft) {
if (m_plan == 0) {
configure_descriptor(m_plan, precision, DFTI_REAL, 1, &nfft);
}
RUN_OR_ASSERT(DftiComputeForward(m_plan.get(), src, dst),
"DftiComputeForward failed.")
RUN_OR_ASSERT(DftiComputeForward(m_plan.get(), src, dst), "DftiComputeForward failed.")
}
inline void inverse(scalar_type* dst, complex_type* src, MKL_LONG nfft) {
if (m_plan == 0) {
configure_descriptor(m_plan, precision, DFTI_REAL, 1, &nfft);
}
RUN_OR_ASSERT(DftiComputeBackward(m_plan.get(), src, dst),
"DftiComputeBackward failed.")
RUN_OR_ASSERT(DftiComputeBackward(m_plan.get(), src, dst), "DftiComputeBackward failed.")
}
inline void forward2(complex_type* dst, complex_type* src, int n0, int n1) {
@@ -123,8 +110,7 @@ struct plan<float> {
MKL_LONG sizes[2] = {n0, n1};
configure_descriptor(m_plan, precision, DFTI_COMPLEX, 2, sizes);
}
RUN_OR_ASSERT(DftiComputeForward(m_plan.get(), src, dst),
"DftiComputeForward failed.")
RUN_OR_ASSERT(DftiComputeForward(m_plan.get(), src, dst), "DftiComputeForward failed.")
}
inline void inverse2(complex_type* dst, complex_type* src, int n0, int n1) {
@@ -132,8 +118,7 @@ struct plan<float> {
MKL_LONG sizes[2] = {n0, n1};
configure_descriptor(m_plan, precision, DFTI_COMPLEX, 2, sizes);
}
RUN_OR_ASSERT(DftiComputeBackward(m_plan.get(), src, dst),
"DftiComputeBackward failed.")
RUN_OR_ASSERT(DftiComputeBackward(m_plan.get(), src, dst), "DftiComputeBackward failed.")
}
};
@@ -152,32 +137,28 @@ struct plan<double> {
if (m_plan == 0) {
configure_descriptor(m_plan, precision, DFTI_COMPLEX, 1, &nfft);
}
RUN_OR_ASSERT(DftiComputeForward(m_plan.get(), src, dst),
"DftiComputeForward failed.")
RUN_OR_ASSERT(DftiComputeForward(m_plan.get(), src, dst), "DftiComputeForward failed.")
}
inline void inverse(complex_type* dst, complex_type* src, MKL_LONG nfft) {
if (m_plan == 0) {
configure_descriptor(m_plan, precision, DFTI_COMPLEX, 1, &nfft);
}
RUN_OR_ASSERT(DftiComputeBackward(m_plan.get(), src, dst),
"DftiComputeBackward failed.")
RUN_OR_ASSERT(DftiComputeBackward(m_plan.get(), src, dst), "DftiComputeBackward failed.")
}
inline void forward(complex_type* dst, scalar_type* src, MKL_LONG nfft) {
if (m_plan == 0) {
configure_descriptor(m_plan, precision, DFTI_REAL, 1, &nfft);
}
RUN_OR_ASSERT(DftiComputeForward(m_plan.get(), src, dst),
"DftiComputeForward failed.")
RUN_OR_ASSERT(DftiComputeForward(m_plan.get(), src, dst), "DftiComputeForward failed.")
}
inline void inverse(scalar_type* dst, complex_type* src, MKL_LONG nfft) {
if (m_plan == 0) {
configure_descriptor(m_plan, precision, DFTI_REAL, 1, &nfft);
}
RUN_OR_ASSERT(DftiComputeBackward(m_plan.get(), src, dst),
"DftiComputeBackward failed.")
RUN_OR_ASSERT(DftiComputeBackward(m_plan.get(), src, dst), "DftiComputeBackward failed.")
}
inline void forward2(complex_type* dst, complex_type* src, int n0, int n1) {
@@ -185,8 +166,7 @@ struct plan<double> {
MKL_LONG sizes[2] = {n0, n1};
configure_descriptor(m_plan, precision, DFTI_COMPLEX, 2, sizes);
}
RUN_OR_ASSERT(DftiComputeForward(m_plan.get(), src, dst),
"DftiComputeForward failed.")
RUN_OR_ASSERT(DftiComputeForward(m_plan.get(), src, dst), "DftiComputeForward failed.")
}
inline void inverse2(complex_type* dst, complex_type* src, int n0, int n1) {
@@ -194,8 +174,7 @@ struct plan<double> {
MKL_LONG sizes[2] = {n0, n1};
configure_descriptor(m_plan, precision, DFTI_COMPLEX, 2, sizes);
}
RUN_OR_ASSERT(DftiComputeBackward(m_plan.get(), src, dst),
"DftiComputeBackward failed.")
RUN_OR_ASSERT(DftiComputeBackward(m_plan.get(), src, dst), "DftiComputeBackward failed.")
}
};
@@ -209,71 +188,53 @@ struct imklfft_impl {
// complex-to-complex forward FFT
inline void fwd(Complex* dst, const Complex* src, int nfft) {
MKL_LONG size = nfft;
get_plan(nfft, dst, src)
.forward(complex_cast(dst), complex_cast(src), size);
get_plan(nfft, dst, src).forward(complex_cast(dst), complex_cast(src), size);
}
// real-to-complex forward FFT
inline void fwd(Complex* dst, const Scalar* src, int nfft) {
MKL_LONG size = nfft;
get_plan(nfft, dst, src)
.forward(complex_cast(dst), const_cast<Scalar*>(src), nfft);
get_plan(nfft, dst, src).forward(complex_cast(dst), const_cast<Scalar*>(src), nfft);
}
// 2-d complex-to-complex
inline void fwd2(Complex* dst, const Complex* src, int n0, int n1) {
get_plan(n0, n1, dst, src)
.forward2(complex_cast(dst), complex_cast(src), n0, n1);
get_plan(n0, n1, dst, src).forward2(complex_cast(dst), complex_cast(src), n0, n1);
}
// inverse complex-to-complex
inline void inv(Complex* dst, const Complex* src, int nfft) {
MKL_LONG size = nfft;
get_plan(nfft, dst, src)
.inverse(complex_cast(dst), complex_cast(src), nfft);
get_plan(nfft, dst, src).inverse(complex_cast(dst), complex_cast(src), nfft);
}
// half-complex to scalar
inline void inv(Scalar* dst, const Complex* src, int nfft) {
MKL_LONG size = nfft;
get_plan(nfft, dst, src)
.inverse(const_cast<Scalar*>(dst), complex_cast(src), nfft);
get_plan(nfft, dst, src).inverse(const_cast<Scalar*>(dst), complex_cast(src), nfft);
}
// 2-d complex-to-complex
inline void inv2(Complex* dst, const Complex* src, int n0, int n1) {
get_plan(n0, n1, dst, src)
.inverse2(complex_cast(dst), complex_cast(src), n0, n1);
get_plan(n0, n1, dst, src).inverse2(complex_cast(dst), complex_cast(src), n0, n1);
}
private:
std::map<int64_t, plan<Scalar>> m_plans;
inline plan<Scalar>& get_plan(int nfft, void* dst,
const void* src) {
inline plan<Scalar>& get_plan(int nfft, void* dst, const void* src) {
int inplace = dst == src ? 1 : 0;
int aligned = ((reinterpret_cast<size_t>(src) & 15) |
(reinterpret_cast<size_t>(dst) & 15)) == 0
? 1
: 0;
int64_t key = ((nfft << 2) | (inplace << 1) | aligned)
<< 1;
int aligned = ((reinterpret_cast<size_t>(src) & 15) | (reinterpret_cast<size_t>(dst) & 15)) == 0 ? 1 : 0;
int64_t key = ((nfft << 2) | (inplace << 1) | aligned) << 1;
// Create element if key does not exist.
return m_plans[key];
}
inline plan<Scalar>& get_plan(int n0, int n1, void* dst,
const void* src) {
inline plan<Scalar>& get_plan(int n0, int n1, void* dst, const void* src) {
int inplace = (dst == src) ? 1 : 0;
int aligned = ((reinterpret_cast<size_t>(src) & 15) |
(reinterpret_cast<size_t>(dst) & 15)) == 0
? 1
: 0;
int64_t key = (((((int64_t)n0) << 31) | (n1 << 2) |
(inplace << 1) | aligned)
<< 1) +
1;
int aligned = ((reinterpret_cast<size_t>(src) & 15) | (reinterpret_cast<size_t>(dst) & 15)) == 0 ? 1 : 0;
int64_t key = (((((int64_t)n0) << 31) | (n1 << 2) | (inplace << 1) | aligned) << 1) + 1;
// Create element if key does not exist.
return m_plans[key];

624
unsupported/Eigen/src/FFT/ei_kissfft_impl.h
View File

@@ -10,16 +10,15 @@
// IWYU pragma: private
#include "./InternalHeaderCheck.h"
namespace Eigen {
namespace Eigen {
namespace internal {
// This FFT implementation was derived from kissfft http:sourceforge.net/projects/kissfft
// Copyright 2003-2009 Mark Borgerding
// This FFT implementation was derived from kissfft http:sourceforge.net/projects/kissfft
// Copyright 2003-2009 Mark Borgerding
template <typename Scalar_>
struct kiss_cpx_fft
{
struct kiss_cpx_fft {
typedef Scalar_ Scalar;
typedef std::complex<Scalar> Complex;
std::vector<Complex> m_twiddles;
@@ -28,425 +27,384 @@ struct kiss_cpx_fft
std::vector<Complex> m_scratchBuf;
bool m_inverse;
inline void make_twiddles(int nfft, bool inverse)
{
using numext::sin;
inline void make_twiddles(int nfft, bool inverse) {
using numext::cos;
using numext::sin;
m_inverse = inverse;
m_twiddles.resize(nfft);
double phinc = 0.25 * double(EIGEN_PI) / nfft;
double phinc = 0.25 * double(EIGEN_PI) / nfft;
Scalar flip = inverse ? Scalar(1) : Scalar(-1);
m_twiddles[0] = Complex(Scalar(1), Scalar(0));
if ((nfft&1)==0)
m_twiddles[nfft/2] = Complex(Scalar(-1), Scalar(0));
int i=1;
for (;i*8<nfft;++i)
{
Scalar c = Scalar(cos(i*8*phinc));
Scalar s = Scalar(sin(i*8*phinc));
m_twiddles[i] = Complex(c, s*flip);
m_twiddles[nfft-i] = Complex(c, -s*flip);
if ((nfft & 1) == 0) m_twiddles[nfft / 2] = Complex(Scalar(-1), Scalar(0));
int i = 1;
for (; i * 8 < nfft; ++i) {
Scalar c = Scalar(cos(i * 8 * phinc));
Scalar s = Scalar(sin(i * 8 * phinc));
m_twiddles[i] = Complex(c, s * flip);
m_twiddles[nfft - i] = Complex(c, -s * flip);
}
for (;i*4<nfft;++i)
{
Scalar c = Scalar(cos((2*nfft-8*i)*phinc));
Scalar s = Scalar(sin((2*nfft-8*i)*phinc));
m_twiddles[i] = Complex(s, c*flip);
m_twiddles[nfft-i] = Complex(s, -c*flip);
for (; i * 4 < nfft; ++i) {
Scalar c = Scalar(cos((2 * nfft - 8 * i) * phinc));
Scalar s = Scalar(sin((2 * nfft - 8 * i) * phinc));
m_twiddles[i] = Complex(s, c * flip);
m_twiddles[nfft - i] = Complex(s, -c * flip);
}
for (;i*8<3*nfft;++i)
{
Scalar c = Scalar(cos((8*i-2*nfft)*phinc));
Scalar s = Scalar(sin((8*i-2*nfft)*phinc));
m_twiddles[i] = Complex(-s, c*flip);
m_twiddles[nfft-i] = Complex(-s, -c*flip);
for (; i * 8 < 3 * nfft; ++i) {
Scalar c = Scalar(cos((8 * i - 2 * nfft) * phinc));
Scalar s = Scalar(sin((8 * i - 2 * nfft) * phinc));
m_twiddles[i] = Complex(-s, c * flip);
m_twiddles[nfft - i] = Complex(-s, -c * flip);
}
for (;i*2<nfft;++i)
{
Scalar c = Scalar(cos((4*nfft-8*i)*phinc));
Scalar s = Scalar(sin((4*nfft-8*i)*phinc));
m_twiddles[i] = Complex(-c, s*flip);
m_twiddles[nfft-i] = Complex(-c, -s*flip);
for (; i * 2 < nfft; ++i) {
Scalar c = Scalar(cos((4 * nfft - 8 * i) * phinc));
Scalar s = Scalar(sin((4 * nfft - 8 * i) * phinc));
m_twiddles[i] = Complex(-c, s * flip);
m_twiddles[nfft - i] = Complex(-c, -s * flip);
}
}
void factorize(int nfft)
{
//start factoring out 4's, then 2's, then 3,5,7,9,...
int n= nfft;
int p=4;
void factorize(int nfft) {
// start factoring out 4's, then 2's, then 3,5,7,9,...
int n = nfft;
int p = 4;
do {
while (n % p) {
switch (p) {
case 4: p = 2; break;
case 2: p = 3; break;
default: p += 2; break;
case 4:
p = 2;
break;
case 2:
p = 3;
break;
default:
p += 2;
break;
}
if (p*p>n)
p=n;// impossible to have a factor > sqrt(n)
if (p * p > n) p = n; // impossible to have a factor > sqrt(n)
}
n /= p;
m_stageRadix.push_back(p);
m_stageRemainder.push_back(n);
if ( p > 5 )
m_scratchBuf.resize(p); // scratchbuf will be needed in bfly_generic
}while(n>1);
if (p > 5) m_scratchBuf.resize(p); // scratchbuf will be needed in bfly_generic
} while (n > 1);
}
template <typename Src_>
inline
void work( int stage,Complex * xout, const Src_ * xin, size_t fstride,size_t in_stride)
{
int p = m_stageRadix[stage];
int m = m_stageRemainder[stage];
Complex * Fout_beg = xout;
Complex * Fout_end = xout + p*m;
inline void work(int stage, Complex *xout, const Src_ *xin, size_t fstride, size_t in_stride) {
int p = m_stageRadix[stage];
int m = m_stageRemainder[stage];
Complex *Fout_beg = xout;
Complex *Fout_end = xout + p * m;
if (m>1) {
do{
// recursive call:
// DFT of size m*p performed by doing
// p instances of smaller DFTs of size m,
// each one takes a decimated version of the input
work(stage+1, xout , xin, fstride*p,in_stride);
xin += fstride*in_stride;
}while( (xout += m) != Fout_end );
}else{
do{
*xout = *xin;
xin += fstride*in_stride;
}while(++xout != Fout_end );
}
xout=Fout_beg;
// recombine the p smaller DFTs
switch (p) {
case 2: bfly2(xout,fstride,m); break;
case 3: bfly3(xout,fstride,m); break;
case 4: bfly4(xout,fstride,m); break;
case 5: bfly5(xout,fstride,m); break;
default: bfly_generic(xout,fstride,m,p); break;
}
if (m > 1) {
do {
// recursive call:
// DFT of size m*p performed by doing
// p instances of smaller DFTs of size m,
// each one takes a decimated version of the input
work(stage + 1, xout, xin, fstride * p, in_stride);
xin += fstride * in_stride;
} while ((xout += m) != Fout_end);
} else {
do {
*xout = *xin;
xin += fstride * in_stride;
} while (++xout != Fout_end);
}
xout = Fout_beg;
inline
void bfly2( Complex * Fout, const size_t fstride, int m)
{
for (int k=0;k<m;++k) {
Complex t = Fout[m+k] * m_twiddles[k*fstride];
Fout[m+k] = Fout[k] - t;
Fout[k] += t;
}
// recombine the p smaller DFTs
switch (p) {
case 2:
bfly2(xout, fstride, m);
break;
case 3:
bfly3(xout, fstride, m);
break;
case 4:
bfly4(xout, fstride, m);
break;
case 5:
bfly5(xout, fstride, m);
break;
default:
bfly_generic(xout, fstride, m, p);
break;
}
}
inline
void bfly4( Complex * Fout, const size_t fstride, const size_t m)
{
Complex scratch[6];
int negative_if_inverse = m_inverse * -2 +1;
for (size_t k=0;k<m;++k) {
scratch[0] = Fout[k+m] * m_twiddles[k*fstride];
scratch[1] = Fout[k+2*m] * m_twiddles[k*fstride*2];
scratch[2] = Fout[k+3*m] * m_twiddles[k*fstride*3];
scratch[5] = Fout[k] - scratch[1];
Fout[k] += scratch[1];
scratch[3] = scratch[0] + scratch[2];
scratch[4] = scratch[0] - scratch[2];
scratch[4] = Complex( scratch[4].imag()*negative_if_inverse , -scratch[4].real()* negative_if_inverse );
Fout[k+2*m] = Fout[k] - scratch[3];
Fout[k] += scratch[3];
Fout[k+m] = scratch[5] + scratch[4];
Fout[k+3*m] = scratch[5] - scratch[4];
}
inline void bfly2(Complex *Fout, const size_t fstride, int m) {
for (int k = 0; k < m; ++k) {
Complex t = Fout[m + k] * m_twiddles[k * fstride];
Fout[m + k] = Fout[k] - t;
Fout[k] += t;
}
}
inline
void bfly3( Complex * Fout, const size_t fstride, const size_t m)
{
size_t k=m;
const size_t m2 = 2*m;
Complex *tw1,*tw2;
Complex scratch[5];
Complex epi3;
epi3 = m_twiddles[fstride*m];
inline void bfly4(Complex *Fout, const size_t fstride, const size_t m) {
Complex scratch[6];
int negative_if_inverse = m_inverse * -2 + 1;
for (size_t k = 0; k < m; ++k) {
scratch[0] = Fout[k + m] * m_twiddles[k * fstride];
scratch[1] = Fout[k + 2 * m] * m_twiddles[k * fstride * 2];
scratch[2] = Fout[k + 3 * m] * m_twiddles[k * fstride * 3];
scratch[5] = Fout[k] - scratch[1];
tw1=tw2=&m_twiddles[0];
Fout[k] += scratch[1];
scratch[3] = scratch[0] + scratch[2];
scratch[4] = scratch[0] - scratch[2];
scratch[4] = Complex(scratch[4].imag() * negative_if_inverse, -scratch[4].real() * negative_if_inverse);
do{
scratch[1]=Fout[m] * *tw1;
scratch[2]=Fout[m2] * *tw2;
scratch[3]=scratch[1]+scratch[2];
scratch[0]=scratch[1]-scratch[2];
tw1 += fstride;
tw2 += fstride*2;
Fout[m] = Complex( Fout->real() - Scalar(.5)*scratch[3].real() , Fout->imag() - Scalar(.5)*scratch[3].imag() );
scratch[0] *= epi3.imag();
*Fout += scratch[3];
Fout[m2] = Complex( Fout[m].real() + scratch[0].imag() , Fout[m].imag() - scratch[0].real() );
Fout[m] += Complex( -scratch[0].imag(),scratch[0].real() );
++Fout;
}while(--k);
Fout[k + 2 * m] = Fout[k] - scratch[3];
Fout[k] += scratch[3];
Fout[k + m] = scratch[5] + scratch[4];
Fout[k + 3 * m] = scratch[5] - scratch[4];
}
}
inline
void bfly5( Complex * Fout, const size_t fstride, const size_t m)
{
Complex *Fout0,*Fout1,*Fout2,*Fout3,*Fout4;
size_t u;
Complex scratch[13];
Complex * twiddles = &m_twiddles[0];
Complex *tw;
Complex ya,yb;
ya = twiddles[fstride*m];
yb = twiddles[fstride*2*m];
inline void bfly3(Complex *Fout, const size_t fstride, const size_t m) {
size_t k = m;
const size_t m2 = 2 * m;
Complex *tw1, *tw2;
Complex scratch[5];
Complex epi3;
epi3 = m_twiddles[fstride * m];
Fout0=Fout;
Fout1=Fout0+m;
Fout2=Fout0+2*m;
Fout3=Fout0+3*m;
Fout4=Fout0+4*m;
tw1 = tw2 = &m_twiddles[0];
tw=twiddles;
for ( u=0; u<m; ++u ) {
scratch[0] = *Fout0;
do {
scratch[1] = Fout[m] * *tw1;
scratch[2] = Fout[m2] * *tw2;
scratch[1] = *Fout1 * tw[u*fstride];
scratch[2] = *Fout2 * tw[2*u*fstride];
scratch[3] = *Fout3 * tw[3*u*fstride];
scratch[4] = *Fout4 * tw[4*u*fstride];
scratch[3] = scratch[1] + scratch[2];
scratch[0] = scratch[1] - scratch[2];
tw1 += fstride;
tw2 += fstride * 2;
Fout[m] = Complex(Fout->real() - Scalar(.5) * scratch[3].real(), Fout->imag() - Scalar(.5) * scratch[3].imag());
scratch[0] *= epi3.imag();
*Fout += scratch[3];
Fout[m2] = Complex(Fout[m].real() + scratch[0].imag(), Fout[m].imag() - scratch[0].real());
Fout[m] += Complex(-scratch[0].imag(), scratch[0].real());
++Fout;
} while (--k);
}
scratch[7] = scratch[1] + scratch[4];
scratch[10] = scratch[1] - scratch[4];
scratch[8] = scratch[2] + scratch[3];
scratch[9] = scratch[2] - scratch[3];
inline void bfly5(Complex *Fout, const size_t fstride, const size_t m) {
Complex *Fout0, *Fout1, *Fout2, *Fout3, *Fout4;
size_t u;
Complex scratch[13];
Complex *twiddles = &m_twiddles[0];
Complex *tw;
Complex ya, yb;
ya = twiddles[fstride * m];
yb = twiddles[fstride * 2 * m];
*Fout0 += scratch[7];
*Fout0 += scratch[8];
Fout0 = Fout;
Fout1 = Fout0 + m;
Fout2 = Fout0 + 2 * m;
Fout3 = Fout0 + 3 * m;
Fout4 = Fout0 + 4 * m;
scratch[5] = scratch[0] + Complex(
(scratch[7].real()*ya.real() ) + (scratch[8].real() *yb.real() ),
(scratch[7].imag()*ya.real()) + (scratch[8].imag()*yb.real())
);
tw = twiddles;
for (u = 0; u < m; ++u) {
scratch[0] = *Fout0;
scratch[6] = Complex(
(scratch[10].imag()*ya.imag()) + (scratch[9].imag()*yb.imag()),
-(scratch[10].real()*ya.imag()) - (scratch[9].real()*yb.imag())
);
scratch[1] = *Fout1 * tw[u * fstride];
scratch[2] = *Fout2 * tw[2 * u * fstride];
scratch[3] = *Fout3 * tw[3 * u * fstride];
scratch[4] = *Fout4 * tw[4 * u * fstride];
*Fout1 = scratch[5] - scratch[6];
*Fout4 = scratch[5] + scratch[6];
scratch[7] = scratch[1] + scratch[4];
scratch[10] = scratch[1] - scratch[4];
scratch[8] = scratch[2] + scratch[3];
scratch[9] = scratch[2] - scratch[3];
scratch[11] = scratch[0] +
Complex(
(scratch[7].real()*yb.real()) + (scratch[8].real()*ya.real()),
(scratch[7].imag()*yb.real()) + (scratch[8].imag()*ya.real())
);
*Fout0 += scratch[7];
*Fout0 += scratch[8];
scratch[12] = Complex(
-(scratch[10].imag()*yb.imag()) + (scratch[9].imag()*ya.imag()),
(scratch[10].real()*yb.imag()) - (scratch[9].real()*ya.imag())
);
scratch[5] = scratch[0] + Complex((scratch[7].real() * ya.real()) + (scratch[8].real() * yb.real()),
(scratch[7].imag() * ya.real()) + (scratch[8].imag() * yb.real()));
*Fout2=scratch[11]+scratch[12];
*Fout3=scratch[11]-scratch[12];
scratch[6] = Complex((scratch[10].imag() * ya.imag()) + (scratch[9].imag() * yb.imag()),
-(scratch[10].real() * ya.imag()) - (scratch[9].real() * yb.imag()));
++Fout0;++Fout1;++Fout2;++Fout3;++Fout4;
}
*Fout1 = scratch[5] - scratch[6];
*Fout4 = scratch[5] + scratch[6];
scratch[11] = scratch[0] + Complex((scratch[7].real() * yb.real()) + (scratch[8].real() * ya.real()),
(scratch[7].imag() * yb.real()) + (scratch[8].imag() * ya.real()));
scratch[12] = Complex(-(scratch[10].imag() * yb.imag()) + (scratch[9].imag() * ya.imag()),
(scratch[10].real() * yb.imag()) - (scratch[9].real() * ya.imag()));
*Fout2 = scratch[11] + scratch[12];
*Fout3 = scratch[11] - scratch[12];
++Fout0;
++Fout1;
++Fout2;
++Fout3;
++Fout4;
}
}
/* perform the butterfly for one stage of a mixed radix FFT */
inline
void bfly_generic(
Complex * Fout,
const size_t fstride,
int m,
int p
)
{
int u,k,q1,q;
Complex * twiddles = &m_twiddles[0];
Complex t;
int Norig = static_cast<int>(m_twiddles.size());
Complex * scratchbuf = &m_scratchBuf[0];
inline void bfly_generic(Complex *Fout, const size_t fstride, int m, int p) {
int u, k, q1, q;
Complex *twiddles = &m_twiddles[0];
Complex t;
int Norig = static_cast<int>(m_twiddles.size());
Complex *scratchbuf = &m_scratchBuf[0];
for ( u=0; u<m; ++u ) {
k=u;
for ( q1=0 ; q1<p ; ++q1 ) {
scratchbuf[q1] = Fout[ k ];
k += m;
}
for (u = 0; u < m; ++u) {
k = u;
for (q1 = 0; q1 < p; ++q1) {
scratchbuf[q1] = Fout[k];
k += m;
}
k=u;
for ( q1=0 ; q1<p ; ++q1 ) {
int twidx=0;
Fout[ k ] = scratchbuf[0];
for (q=1;q<p;++q ) {
twidx += static_cast<int>(fstride) * k;
if (twidx>=Norig) twidx-=Norig;
t=scratchbuf[q] * twiddles[twidx];
Fout[ k ] += t;
}
k += m;
k = u;
for (q1 = 0; q1 < p; ++q1) {
int twidx = 0;
Fout[k] = scratchbuf[0];
for (q = 1; q < p; ++q) {
twidx += static_cast<int>(fstride) * k;
if (twidx >= Norig) twidx -= Norig;
t = scratchbuf[q] * twiddles[twidx];
Fout[k] += t;
}
k += m;
}
}
}
};
template <typename Scalar_>
struct kissfft_impl
{
struct kissfft_impl {
typedef Scalar_ Scalar;
typedef std::complex<Scalar> Complex;
void clear()
{
void clear() {
m_plans.clear();
m_realTwiddles.clear();
}
inline
void fwd( Complex * dst,const Complex *src,int nfft)
{
get_plan(nfft,false).work(0, dst, src, 1,1);
}
inline void fwd(Complex *dst, const Complex *src, int nfft) { get_plan(nfft, false).work(0, dst, src, 1, 1); }
inline
void fwd2( Complex * dst,const Complex *src,int n0,int n1)
{
EIGEN_UNUSED_VARIABLE(dst);
EIGEN_UNUSED_VARIABLE(src);
EIGEN_UNUSED_VARIABLE(n0);
EIGEN_UNUSED_VARIABLE(n1);
}
inline void fwd2(Complex *dst, const Complex *src, int n0, int n1) {
EIGEN_UNUSED_VARIABLE(dst);
EIGEN_UNUSED_VARIABLE(src);
EIGEN_UNUSED_VARIABLE(n0);
EIGEN_UNUSED_VARIABLE(n1);
}
inline
void inv2( Complex * dst,const Complex *src,int n0,int n1)
{
EIGEN_UNUSED_VARIABLE(dst);
EIGEN_UNUSED_VARIABLE(src);
EIGEN_UNUSED_VARIABLE(n0);
EIGEN_UNUSED_VARIABLE(n1);
}
inline void inv2(Complex *dst, const Complex *src, int n0, int n1) {
EIGEN_UNUSED_VARIABLE(dst);
EIGEN_UNUSED_VARIABLE(src);
EIGEN_UNUSED_VARIABLE(n0);
EIGEN_UNUSED_VARIABLE(n1);
}
// real-to-complex forward FFT
// perform two FFTs of src even and src odd
// then twiddle to recombine them into the half-spectrum format
// then fill in the conjugate symmetric half
inline
void fwd( Complex * dst,const Scalar * src,int nfft)
{
if ( nfft&3 ) {
// use generic mode for odd
m_tmpBuf1.resize(nfft);
get_plan(nfft,false).work(0, &m_tmpBuf1[0], src, 1,1);
std::copy(m_tmpBuf1.begin(),m_tmpBuf1.begin()+(nfft>>1)+1,dst );
}else{
int ncfft = nfft>>1;
int ncfft2 = nfft>>2;
Complex * rtw = real_twiddles(ncfft2);
inline void fwd(Complex *dst, const Scalar *src, int nfft) {
if (nfft & 3) {
// use generic mode for odd
m_tmpBuf1.resize(nfft);
get_plan(nfft, false).work(0, &m_tmpBuf1[0], src, 1, 1);
std::copy(m_tmpBuf1.begin(), m_tmpBuf1.begin() + (nfft >> 1) + 1, dst);
} else {
int ncfft = nfft >> 1;
int ncfft2 = nfft >> 2;
Complex *rtw = real_twiddles(ncfft2);
// use optimized mode for even real
fwd( dst, reinterpret_cast<const Complex*> (src), ncfft);
Complex dc(dst[0].real() + dst[0].imag());
Complex nyquist(dst[0].real() - dst[0].imag());
int k;
for ( k=1;k <= ncfft2 ; ++k ) {
Complex fpk = dst[k];
Complex fpnk = conj(dst[ncfft-k]);
Complex f1k = fpk + fpnk;
Complex f2k = fpk - fpnk;
Complex tw= f2k * rtw[k-1];
dst[k] = (f1k + tw) * Scalar(.5);
dst[ncfft-k] = conj(f1k -tw)*Scalar(.5);
}
dst[0] = dc;
dst[ncfft] = nyquist;
// use optimized mode for even real
fwd(dst, reinterpret_cast<const Complex *>(src), ncfft);
Complex dc(dst[0].real() + dst[0].imag());
Complex nyquist(dst[0].real() - dst[0].imag());
int k;
for (k = 1; k <= ncfft2; ++k) {
Complex fpk = dst[k];
Complex fpnk = conj(dst[ncfft - k]);
Complex f1k = fpk + fpnk;
Complex f2k = fpk - fpnk;
Complex tw = f2k * rtw[k - 1];
dst[k] = (f1k + tw) * Scalar(.5);
dst[ncfft - k] = conj(f1k - tw) * Scalar(.5);
}
dst[0] = dc;
dst[ncfft] = nyquist;
}
}
// inverse complex-to-complex
inline
void inv(Complex * dst,const Complex *src,int nfft)
{
get_plan(nfft,true).work(0, dst, src, 1,1);
}
inline void inv(Complex *dst, const Complex *src, int nfft) { get_plan(nfft, true).work(0, dst, src, 1, 1); }
// half-complex to scalar
inline
void inv( Scalar * dst,const Complex * src,int nfft)
{
if (nfft&3) {
m_tmpBuf1.resize(nfft);
m_tmpBuf2.resize(nfft);
std::copy(src,src+(nfft>>1)+1,m_tmpBuf1.begin() );
for (int k=1;k<(nfft>>1)+1;++k)
m_tmpBuf1[nfft-k] = conj(m_tmpBuf1[k]);
inv(&m_tmpBuf2[0],&m_tmpBuf1[0],nfft);
for (int k=0;k<nfft;++k)
dst[k] = m_tmpBuf2[k].real();
}else{
// optimized version for multiple of 4
int ncfft = nfft>>1;
int ncfft2 = nfft>>2;
Complex * rtw = real_twiddles(ncfft2);
m_tmpBuf1.resize(ncfft);
m_tmpBuf1[0] = Complex( src[0].real() + src[ncfft].real(), src[0].real() - src[ncfft].real() );
for (int k = 1; k <= ncfft / 2; ++k) {
Complex fk = src[k];
Complex fnkc = conj(src[ncfft-k]);
Complex fek = fk + fnkc;
Complex tmp = fk - fnkc;
Complex fok = tmp * conj(rtw[k-1]);
m_tmpBuf1[k] = fek + fok;
m_tmpBuf1[ncfft-k] = conj(fek - fok);
}
get_plan(ncfft,true).work(0, reinterpret_cast<Complex*>(dst), &m_tmpBuf1[0], 1,1);
inline void inv(Scalar *dst, const Complex *src, int nfft) {
if (nfft & 3) {
m_tmpBuf1.resize(nfft);
m_tmpBuf2.resize(nfft);
std::copy(src, src + (nfft >> 1) + 1, m_tmpBuf1.begin());
for (int k = 1; k < (nfft >> 1) + 1; ++k) m_tmpBuf1[nfft - k] = conj(m_tmpBuf1[k]);
inv(&m_tmpBuf2[0], &m_tmpBuf1[0], nfft);
for (int k = 0; k < nfft; ++k) dst[k] = m_tmpBuf2[k].real();
} else {
// optimized version for multiple of 4
int ncfft = nfft >> 1;
int ncfft2 = nfft >> 2;
Complex *rtw = real_twiddles(ncfft2);
m_tmpBuf1.resize(ncfft);
m_tmpBuf1[0] = Complex(src[0].real() + src[ncfft].real(), src[0].real() - src[ncfft].real());
for (int k = 1; k <= ncfft / 2; ++k) {
Complex fk = src[k];
Complex fnkc = conj(src[ncfft - k]);
Complex fek = fk + fnkc;
Complex tmp = fk - fnkc;
Complex fok = tmp * conj(rtw[k - 1]);
m_tmpBuf1[k] = fek + fok;
m_tmpBuf1[ncfft - k] = conj(fek - fok);
}
get_plan(ncfft, true).work(0, reinterpret_cast<Complex *>(dst), &m_tmpBuf1[0], 1, 1);
}
}
protected:
protected:
typedef kiss_cpx_fft<Scalar> PlanData;
typedef std::map<int,PlanData> PlanMap;
typedef std::map<int, PlanData> PlanMap;
PlanMap m_plans;
std::map<int, std::vector<Complex> > m_realTwiddles;
std::vector<Complex> m_tmpBuf1;
std::vector<Complex> m_tmpBuf2;
inline
int PlanKey(int nfft, bool isinverse) const { return (nfft<<1) | int(isinverse); }
inline int PlanKey(int nfft, bool isinverse) const { return (nfft << 1) | int(isinverse); }
inline
PlanData & get_plan(int nfft, bool inverse)
{
// TODO look for PlanKey(nfft, ! inverse) and conjugate the twiddles
PlanData & pd = m_plans[ PlanKey(nfft,inverse) ];
if ( pd.m_twiddles.size() == 0 ) {
pd.make_twiddles(nfft,inverse);
pd.factorize(nfft);
}
return pd;
inline PlanData &get_plan(int nfft, bool inverse) {
// TODO look for PlanKey(nfft, ! inverse) and conjugate the twiddles
PlanData &pd = m_plans[PlanKey(nfft, inverse)];
if (pd.m_twiddles.size() == 0) {
pd.make_twiddles(nfft, inverse);
pd.factorize(nfft);
}
return pd;
}
inline
Complex * real_twiddles(int ncfft2)
{
using std::acos;
std::vector<Complex> & twidref = m_realTwiddles[ncfft2];// creates new if not there
if ( (int)twidref.size() != ncfft2 ) {
twidref.resize(ncfft2);
int ncfft= ncfft2<<1;
Scalar pi = acos( Scalar(-1) );
for (int k=1;k<=ncfft2;++k)
twidref[k-1] = exp( Complex(0,-pi * (Scalar(k) / ncfft + Scalar(.5)) ) );
}
return &twidref[0];
inline Complex *real_twiddles(int ncfft2) {
using std::acos;
std::vector<Complex> &twidref = m_realTwiddles[ncfft2]; // creates new if not there
if ((int)twidref.size() != ncfft2) {
twidref.resize(ncfft2);
int ncfft = ncfft2 << 1;
Scalar pi = acos(Scalar(-1));
for (int k = 1; k <= ncfft2; ++k) twidref[k - 1] = exp(Complex(0, -pi * (Scalar(k) / ncfft + Scalar(.5))));
}
return &twidref[0];
}
};
} // end namespace internal
} // end namespace internal
} // end namespace Eigen
} // end namespace Eigen

65
unsupported/Eigen/src/FFT/ei_pocketfft_impl.h
View File

@@ -1,5 +1,5 @@
// This file is part of Eigen, a lightweight C++ template library
// for linear algebra.
// for linear algebra.
//
// This Source Code Form is subject to the terms of the Mozilla
// Public License v. 2.0. If a copy of the MPL was not distributed
@@ -12,58 +12,57 @@ namespace Eigen {
namespace internal {
template<typename _Scalar>
struct pocketfft_impl
{
template <typename _Scalar>
struct pocketfft_impl {
typedef _Scalar Scalar;
typedef std::complex<Scalar> Complex;
inline void clear() {}
inline void fwd(Complex* dst, const Scalar* src, int nfft){
const shape_t shape_{ static_cast<size_t>(nfft) };
const shape_t axes_{ 0 };
const stride_t stride_in{ sizeof(Scalar) };
const stride_t stride_out{ sizeof(Complex) };
inline void fwd(Complex* dst, const Scalar* src, int nfft) {
const shape_t shape_{static_cast<size_t>(nfft)};
const shape_t axes_{0};
const stride_t stride_in{sizeof(Scalar)};
const stride_t stride_out{sizeof(Complex)};
r2c(shape_, stride_in, stride_out, axes_, FORWARD, src, dst, static_cast<Scalar>(1));
}
inline void fwd(Complex* dst, const Complex* src, int nfft){
const shape_t shape_{ static_cast<size_t>(nfft) };
const shape_t axes_{ 0 };
const stride_t stride_{ sizeof(Complex) };
inline void fwd(Complex* dst, const Complex* src, int nfft) {
const shape_t shape_{static_cast<size_t>(nfft)};
const shape_t axes_{0};
const stride_t stride_{sizeof(Complex)};
c2c(shape_, stride_, stride_, axes_, FORWARD, src, dst, static_cast<Scalar>(1));
}
inline void inv(Scalar* dst, const Complex* src, int nfft){
const shape_t shape_{ static_cast<size_t>(nfft) };
const shape_t axes_{ 0 };
const stride_t stride_in{ sizeof(Complex) };
const stride_t stride_out{ sizeof(Scalar) };
inline void inv(Scalar* dst, const Complex* src, int nfft) {
const shape_t shape_{static_cast<size_t>(nfft)};
const shape_t axes_{0};
const stride_t stride_in{sizeof(Complex)};
const stride_t stride_out{sizeof(Scalar)};
c2r(shape_, stride_in, stride_out, axes_, BACKWARD, src, dst, static_cast<Scalar>(1));
}
}
inline void inv(Complex* dst, const Complex* src, int nfft){
const shape_t shape_{ static_cast<size_t>(nfft) };
const shape_t axes_{ 0 };
const stride_t stride_{ sizeof(Complex) };
inline void inv(Complex* dst, const Complex* src, int nfft) {
const shape_t shape_{static_cast<size_t>(nfft)};
const shape_t axes_{0};
const stride_t stride_{sizeof(Complex)};
c2c(shape_, stride_, stride_, axes_, BACKWARD, src, dst, static_cast<Scalar>(1));
}
inline void fwd2(Complex* dst, const Complex* src, int nfft0, int nfft1){
const shape_t shape_{ static_cast<size_t>(nfft0), static_cast<size_t>(nfft1) };
const shape_t axes_{ 0, 1 };
const stride_t stride_{ static_cast<ptrdiff_t>(sizeof(Complex)*nfft1), static_cast<ptrdiff_t>(sizeof(Complex)) };
inline void fwd2(Complex* dst, const Complex* src, int nfft0, int nfft1) {
const shape_t shape_{static_cast<size_t>(nfft0), static_cast<size_t>(nfft1)};
const shape_t axes_{0, 1};
const stride_t stride_{static_cast<ptrdiff_t>(sizeof(Complex) * nfft1), static_cast<ptrdiff_t>(sizeof(Complex))};
c2c(shape_, stride_, stride_, axes_, FORWARD, src, dst, static_cast<Scalar>(1));
}
inline void inv2(Complex* dst, const Complex* src, int nfft0, int nfft1){
const shape_t shape_{ static_cast<size_t>(nfft0), static_cast<size_t>(nfft1) };
const shape_t axes_{ 0, 1 };
const stride_t stride_{ static_cast<ptrdiff_t>(sizeof(Complex)*nfft1), static_cast<ptrdiff_t>(sizeof(Complex)) };
inline void inv2(Complex* dst, const Complex* src, int nfft0, int nfft1) {
const shape_t shape_{static_cast<size_t>(nfft0), static_cast<size_t>(nfft1)};
const shape_t axes_{0, 1};
const stride_t stride_{static_cast<ptrdiff_t>(sizeof(Complex) * nfft1), static_cast<ptrdiff_t>(sizeof(Complex))};
c2c(shape_, stride_, stride_, axes_, BACKWARD, src, dst, static_cast<Scalar>(1));
}
};
} // namespace internal
} // namespace Eigen
} // namespace internal
} // namespace Eigen