devtools/eigen
1
0
Fork 0
mirror of https://gitlab.com/libeigen/eigen.git synced 2026-04-10 11:34:33 +08:00
Code Issues Packages Projects Releases Wiki Activity

Vectorize BLAS level 1/2 routines with Eigen expressions

Browse Source 
libeigen/eigen!2404

Co-authored-by: Rasmus Munk Larsen <rmlarsen@gmail.com>
This commit is contained in:
Rasmus Munk Larsen
2026-04-05 18:53:11 -07:00
parent 4ad90a60f1
commit 8eabfb5342
6 changed files with 294 additions and 204 deletions
Download Patch File Download Diff File Expand all files Collapse all files

165
blas/level2_real_impl.h
View File

@@ -194,62 +194,79 @@ EIGEN_BLAS_FUNC(sbmv)
if (*n == 0 || (alpha == Scalar(0) && beta == Scalar(1))) return;
int kx = *incx > 0 ? 0 : (1 - *n) * *incx;
int ky = *incy > 0 ? 0 : (1 - *n) * *incy;
const Scalar *actual_x = get_compact_vector(x, *n, *incx);
Scalar *actual_y = get_compact_vector(y, *n, *incy);
// First form y := beta*y.
if (beta != Scalar(1)) {
int iy = ky;
for (int i = 0; i < *n; ++i) {
y[iy] = (beta == Scalar(0)) ? Scalar(0) : beta * y[iy];
iy += *incy;
}
if (beta == Scalar(0))
make_vector(actual_y, *n).setZero();
else
make_vector(actual_y, *n) *= beta;
}
if (alpha == Scalar(0)) return;
if (alpha == Scalar(0)) {
if (actual_x != x) delete[] actual_x;
if (actual_y != y) delete[] copy_back(actual_y, y, *n, *incy);
return;
}
if (UPLO(*uplo) == UP) {
// Upper triangle: A[i,j] at a[(k+i-j) + j*lda], diagonal at row k.
int jx = kx, jy = ky;
for (int j = 0; j < *n; ++j) {
Scalar temp1 = alpha * x[jx];
Scalar temp2 = Scalar(0);
int ix = kx, iy = ky;
for (int i = std::max(0, j - *k); i < j; ++i) {
Scalar aij = a[(*k + i - j) + j * *lda];
y[iy] += temp1 * aij;
temp2 += aij * x[ix];
ix += *incx;
iy += *incy;
if (*k >= 8) {
// Vectorized path: use Eigen Map segments for the inner band operations.
ConstMatrixType band(a, *k + 1, *n, *lda);
if (UPLO(*uplo) == UP) {
for (int j = 0; j < *n; ++j) {
int start = std::max(0, j - *k);
int len = j - start;
int offset = *k - (j - start);
Scalar temp1 = alpha * actual_x[j];
actual_y[j] += temp1 * band(*k, j);
if (len > 0) {
make_vector(actual_y + start, len) += temp1 * band.col(j).segment(offset, len);
actual_y[j] += alpha * band.col(j).segment(offset, len).dot(make_vector(actual_x + start, len));
}
}
y[jy] += temp1 * a[*k + j * *lda] + alpha * temp2;
jx += *incx;
jy += *incy;
if (j >= *k) {
kx += *incx;
ky += *incy;
} else {
for (int j = 0; j < *n; ++j) {
int len = std::min(*n - 1, j + *k) - j;
Scalar temp1 = alpha * actual_x[j];
actual_y[j] += temp1 * band(0, j);
if (len > 0) {
make_vector(actual_y + j + 1, len) += temp1 * band.col(j).segment(1, len);
actual_y[j] += alpha * band.col(j).segment(1, len).dot(make_vector(actual_x + j + 1, len));
}
}
}
} else {
// Lower triangle: A[i,j] at a[(i-j) + j*lda], diagonal at row 0.
int jx = kx, jy = ky;
for (int j = 0; j < *n; ++j) {
Scalar temp1 = alpha * x[jx];
Scalar temp2 = Scalar(0);
y[jy] += temp1 * a[j * *lda];
int ix = jx, iy = jy;
for (int i = j + 1; i <= std::min(*n - 1, j + *k); ++i) {
ix += *incx;
iy += *incy;
Scalar aij = a[(i - j) + j * *lda];
y[iy] += temp1 * aij;
temp2 += aij * x[ix];
// Scalar path: for narrow bandwidth, avoid Map overhead.
if (UPLO(*uplo) == UP) {
for (int j = 0; j < *n; ++j) {
Scalar temp1 = alpha * actual_x[j];
Scalar temp2 = Scalar(0);
for (int i = std::max(0, j - *k); i < j; ++i) {
Scalar aij = a[(*k + i - j) + j * *lda];
actual_y[i] += temp1 * aij;
temp2 += aij * actual_x[i];
}
actual_y[j] += temp1 * a[*k + j * *lda] + alpha * temp2;
}
} else {
for (int j = 0; j < *n; ++j) {
Scalar temp1 = alpha * actual_x[j];
Scalar temp2 = Scalar(0);
actual_y[j] += temp1 * a[j * *lda];
for (int i = j + 1; i <= std::min(*n - 1, j + *k); ++i) {
Scalar aij = a[(i - j) + j * *lda];
actual_y[i] += temp1 * aij;
temp2 += aij * actual_x[i];
}
actual_y[j] += alpha * temp2;
}
y[jy] += alpha * temp2;
jx += *incx;
jy += *incy;
}
}
if (actual_x != x) delete[] actual_x;
if (actual_y != y) delete[] copy_back(actual_y, y, *n, *incy);
}
/** SPMV performs the matrix-vector operation
@@ -285,59 +302,51 @@ EIGEN_BLAS_FUNC(spmv)
if (*n == 0 || (alpha == Scalar(0) && beta == Scalar(1))) return;
int kx = *incx > 0 ? 0 : (1 - *n) * *incx;
int ky = *incy > 0 ? 0 : (1 - *n) * *incy;
const Scalar *actual_x = get_compact_vector(x, *n, *incx);
Scalar *actual_y = get_compact_vector(y, *n, *incy);
// First form y := beta*y.
if (beta != Scalar(1)) {
int iy = ky;
for (int i = 0; i < *n; ++i) {
y[iy] = (beta == Scalar(0)) ? Scalar(0) : beta * y[iy];
iy += *incy;
}
if (beta == Scalar(0))
make_vector(actual_y, *n).setZero();
else
make_vector(actual_y, *n) *= beta;
}
if (alpha == Scalar(0)) return;
if (alpha == Scalar(0)) {
if (actual_x != x) delete[] actual_x;
if (actual_y != y) delete[] copy_back(actual_y, y, *n, *incy);
return;
}
int kk = 0;
if (UPLO(*uplo) == UP) {
// Upper triangle packed.
int jx = kx, jy = ky;
// Upper triangle packed: column j occupies ap[kk..kk+j].
for (int j = 0; j < *n; ++j) {
Scalar temp1 = alpha * x[jx];
Scalar temp2 = Scalar(0);
int ix = kx, iy = ky;
for (int i = 0; i < j; ++i) {
y[iy] += temp1 * ap[kk + i];
temp2 += ap[kk + i] * x[ix];
ix += *incx;
iy += *incy;
Scalar temp1 = alpha * actual_x[j];
actual_y[j] += temp1 * ap[kk + j];
if (j > 0) {
make_vector(actual_y, j) += temp1 * make_vector(ap + kk, j);
actual_y[j] += alpha * make_vector(ap + kk, j).dot(make_vector(actual_x, j));
}
y[jy] += temp1 * ap[kk + j] + alpha * temp2;
jx += *incx;
jy += *incy;
kk += j + 1;
}
} else {
// Lower triangle packed.
int jx = kx, jy = ky;
// Lower triangle packed: column j occupies ap[kk..kk+(n-j-1)].
for (int j = 0; j < *n; ++j) {
Scalar temp1 = alpha * x[jx];
Scalar temp2 = Scalar(0);
y[jy] += temp1 * ap[kk];
int ix = jx, iy = jy;
for (int i = 1; i < *n - j; ++i) {
ix += *incx;
iy += *incy;
y[iy] += temp1 * ap[kk + i];
temp2 += ap[kk + i] * x[ix];
int len = *n - j - 1;
Scalar temp1 = alpha * actual_x[j];
actual_y[j] += temp1 * ap[kk];
if (len > 0) {
make_vector(actual_y + j + 1, len) += temp1 * make_vector(ap + kk + 1, len);
actual_y[j] += alpha * make_vector(ap + kk + 1, len).dot(make_vector(actual_x + j + 1, len));
}
y[jy] += alpha * temp2;
jx += *incx;
jy += *incy;
kk += *n - j;
}
}
if (actual_x != x) delete[] actual_x;
if (actual_y != y) delete[] copy_back(actual_y, y, *n, *incy);
}
/** DSPR performs the symmetric rank 1 operation