diff options
author | Cedric Nugteren <web@cedricnugteren.nl> | 2016-03-02 21:18:01 +0100 |
---|---|---|
committer | Cedric Nugteren <web@cedricnugteren.nl> | 2016-03-02 21:18:01 +0100 |
commit | 60da54da5d8cb8dc763c13ba48ec6d8e557a609d (patch) | |
tree | 5c71017dd8280ddfaf7955d621bfd446d8578c1b /src | |
parent | fa79720557412cad605589301580ccda39edce6c (diff) |
Added preliminary support for xHER2 and xSYR2 routines
Diffstat (limited to 'src')
-rw-r--r-- | src/clblast.cc | 54 | ||||
-rw-r--r-- | src/kernels/common.opencl | 7 | ||||
-rw-r--r-- | src/kernels/level2/level2.opencl | 57 | ||||
-rw-r--r-- | src/kernels/level2/xher2.opencl | 104 | ||||
-rw-r--r-- | src/routines/level2/xher2.cc | 114 | ||||
-rw-r--r-- | src/routines/level2/xhpr.cc | 2 | ||||
-rw-r--r-- | src/routines/level2/xspr.cc | 2 | ||||
-rw-r--r-- | src/routines/level2/xsyr.cc | 2 | ||||
-rw-r--r-- | src/routines/level2/xsyr2.cc | 52 |
9 files changed, 362 insertions, 32 deletions
diff --git a/src/clblast.cc b/src/clblast.cc index 466de83e..6b8ac409 100644 --- a/src/clblast.cc +++ b/src/clblast.cc @@ -43,8 +43,10 @@ #include "internal/routines/level2/xgerc.h" #include "internal/routines/level2/xher.h" #include "internal/routines/level2/xhpr.h" +#include "internal/routines/level2/xher2.h" #include "internal/routines/level2/xsyr.h" #include "internal/routines/level2/xspr.h" +#include "internal/routines/level2/xsyr2.h" // BLAS level-3 includes #include "internal/routines/level3/xgemm.h" @@ -1014,14 +1016,24 @@ template StatusCode Hpr<double>(const Layout, const Triangle, // Hermitian rank-2 matrix update: CHER2/ZHER2 template <typename T> -StatusCode Her2(const Layout, const Triangle, - const size_t, - const T, - const cl_mem, const size_t, const size_t, - const cl_mem, const size_t, const size_t, - cl_mem, const size_t, const size_t, - cl_command_queue*, cl_event*) { - return StatusCode::kNotImplemented; +StatusCode Her2(const Layout layout, const Triangle triangle, + const size_t n, + const T alpha, + const cl_mem x_buffer, const size_t x_offset, const size_t x_inc, + const cl_mem y_buffer, const size_t y_offset, const size_t y_inc, + cl_mem a_buffer, const size_t a_offset, const size_t a_ld, + cl_command_queue* queue, cl_event* event) { + auto queue_cpp = Queue(*queue); + auto event_cpp = Event(*event); + auto routine = Xher2<T>(queue_cpp, event_cpp); + auto status = routine.SetUp(); + if (status != StatusCode::kSuccess) { return status; } + return routine.DoHer2(layout, triangle, + n, + alpha, + Buffer<T>(x_buffer), x_offset, x_inc, + Buffer<T>(y_buffer), y_offset, y_inc, + Buffer<T>(a_buffer), a_offset, a_ld); } template StatusCode Her2<float2>(const Layout, const Triangle, const size_t, @@ -1130,14 +1142,24 @@ template StatusCode Spr<double>(const Layout, const Triangle, // Symmetric rank-2 matrix update: SSYR2/DSYR2 template <typename T> -StatusCode Syr2(const Layout, const Triangle, - const size_t, - const T, - const cl_mem, const size_t, const size_t, - const cl_mem, const size_t, const size_t, - cl_mem, const size_t, const size_t, - cl_command_queue*, cl_event*) { - return StatusCode::kNotImplemented; +StatusCode Syr2(const Layout layout, const Triangle triangle, + const size_t n, + const T alpha, + const cl_mem x_buffer, const size_t x_offset, const size_t x_inc, + const cl_mem y_buffer, const size_t y_offset, const size_t y_inc, + cl_mem a_buffer, const size_t a_offset, const size_t a_ld, + cl_command_queue* queue, cl_event* event) { + auto queue_cpp = Queue(*queue); + auto event_cpp = Event(*event); + auto routine = Xsyr2<T>(queue_cpp, event_cpp); + auto status = routine.SetUp(); + if (status != StatusCode::kSuccess) { return status; } + return routine.DoSyr2(layout, triangle, + n, + alpha, + Buffer<T>(x_buffer), x_offset, x_inc, + Buffer<T>(y_buffer), y_offset, y_inc, + Buffer<T>(a_buffer), a_offset, a_ld); } template StatusCode Syr2<float>(const Layout, const Triangle, const size_t, diff --git a/src/kernels/common.opencl b/src/kernels/common.opencl index 973c123e..f2a2e7a7 100644 --- a/src/kernels/common.opencl +++ b/src/kernels/common.opencl @@ -147,13 +147,6 @@ R"( #define AXPBY(e, a, b, c, d) e = a*b + c*d #endif -// The scalar GER function -#if PRECISION == 3232 || PRECISION == 6464 - #define GER(e, a, b, c, d) real ab; ab.x = MulReal(a,b); ab.y = MulImag(a,b); e.x = MulReal(ab,c) + d.x; e.y = MulImag(ab,c) + d.y -#else - #define GER(e, a, b, c, d) e = a*b*c + d -#endif - // The complex conjugate operation for complex transforms #if PRECISION == 3232 || PRECISION == 6464 #define COMPLEX_CONJUGATE(value) value.x = value.x; value.y = -value.y diff --git a/src/kernels/level2/level2.opencl b/src/kernels/level2/level2.opencl index ad92595a..1b0efeab 100644 --- a/src/kernels/level2/level2.opencl +++ b/src/kernels/level2/level2.opencl @@ -39,10 +39,7 @@ inline real LoadVector(const int id, const int max, if (id < max) { real result = gm[id*inc + offset]; if (do_conjugate) { - #if defined(ROUTINE_GERC) - COMPLEX_CONJUGATE(result); - #endif - #if defined(ROUTINE_HER) || defined(ROUTINE_HPR) + #if defined(ROUTINE_GERC) || defined(ROUTINE_HER) || defined(ROUTINE_HPR) || defined(ROUTINE_HER2) || defined(ROUTINE_HPR2) COMPLEX_CONJUGATE(result); #endif } @@ -81,8 +78,16 @@ inline void MatrixUpdate(const int id1, const int id2, const int max1, const int const real avalue = agm[a_index]; // Computes result = alpha * x[i] * y[j] + a[i][j] - real result; - GER(result, alpha, xvalue, yvalue, avalue); + #if PRECISION == 3232 || PRECISION == 6464 + real ax; + ax.x = MulReal(alpha, xvalue); + ax.y = MulImag(alpha, xvalue); + real result; + result.x = MulReal(ax, yvalue) + avalue.x; + result.y = MulImag(ax, yvalue) + avalue.y; + #else + real result = alpha * xvalue * yvalue + avalue; + #endif // For hermetian matrices #if defined(ROUTINE_HER) || defined(ROUTINE_HPR) @@ -94,6 +99,46 @@ inline void MatrixUpdate(const int id1, const int id2, const int max1, const int } } +// Performs the rank-2 matrix update +inline void MatrixUpdate2(const int id1, const int id2, const int max1, const int max2, + __global real* agm, const int a_offset, const int a_ld, + const real alpha1, const real xvalue, const real yvalue, + const real alpha2, const real xtvalue, const real ytvalue, + const int is_upper) { + + // Bounds of a regular matrix + if (id1 < max1 && id2 < max2) { + + const int a_index = id2*a_ld + id1 + a_offset; + + // Loads the current value of the A matrix + const real avalue = agm[a_index]; + + // Computes result = alpha * x[i] * y[j] + alpha * x[j] * y[i] + a[i][j] + #if PRECISION == 3232 || PRECISION == 6464 + real ax; + ax.x = MulReal(alpha2, xvalue); + ax.y = MulImag(alpha2, xvalue); + real atx; + atx.x = MulReal(alpha1, xtvalue); + atx.y = MulImag(alpha1, xtvalue); + real result; + result.x = MulReal(ax, yvalue) + MulReal(atx, ytvalue) + avalue.x; + result.y = MulImag(ax, yvalue) + MulImag(atx, ytvalue) + avalue.y; + #else + real result = alpha1 * xvalue * yvalue + alpha2 * xtvalue * ytvalue + avalue; + #endif + + // For hermetian matrices + #if defined(ROUTINE_HER2) || defined(ROUTINE_HPR2) + if (id1 == id2) { result.y = ZERO; } + #endif + + // Stores the final result + agm[a_index] = result; + } +} + // ================================================================================================= // End of the C++11 raw string literal diff --git a/src/kernels/level2/xher2.opencl b/src/kernels/level2/xher2.opencl new file mode 100644 index 00000000..4a2edce8 --- /dev/null +++ b/src/kernels/level2/xher2.opencl @@ -0,0 +1,104 @@ + +// ================================================================================================= +// This file is part of the CLBlast project. The project is licensed under Apache Version 2.0. This +// project loosely follows the Google C++ styleguide and uses a tab-size of two spaces and a max- +// width of 100 characters per line. +// +// Author(s): +// Cedric Nugteren <www.cedricnugteren.nl> +// +// This file contains the Xher2 kernels for rank-2 matrix update. +// +// ================================================================================================= + +// Enables loading of this file using the C++ pre-processor's #include (C++11 standard raw string +// literal). Comment-out this line for syntax-highlighting when developing. +R"( + +// ================================================================================================= + +// Symmetric version of the rank-2 matrix update kernel (HER2, HPR2, SYR2, SPR2) +__attribute__((reqd_work_group_size(WGS1, WGS2, 1))) +__kernel void Xher2(const int n, const real alpha, + const __global real* restrict xgm, const int x_offset, const int x_inc, + const __global real* restrict ygm, const int y_offset, const int y_inc, + __global real* restrict agm, const int a_offset, const int a_ld, + const int is_upper, const int is_rowmajor) { + + // Register storage for X and Y + real xvalues[WPT]; + real yvalues[WPT]; + real xtvalues[WPT]; + real ytvalues[WPT]; + + // Loads the X-vector + #pragma unroll + for (int w=0; w<WPT; ++w) { + const int id2 = w*get_global_size(1) + get_global_id(1); + xvalues[w] = LoadVector(id2, n, xgm, x_offset, x_inc, !is_rowmajor); + } + + // Loads the X-transposed-vector + #pragma unroll + for (int w=0; w<WPT; ++w) { + const int id1 = w*get_global_size(0) + get_global_id(0); + xtvalues[w] = LoadVector(id1, n, xgm, x_offset, x_inc, is_rowmajor); + } + + // Loads the Y-vector + #pragma unroll + for (int w=0; w<WPT; ++w) { + const int id1 = w*get_global_size(0) + get_global_id(0); + yvalues[w] = LoadVector(id1, n, ygm, y_offset, y_inc, is_rowmajor); + } + + // Loads the Y-transposed-vector + #pragma unroll + for (int w=0; w<WPT; ++w) { + const int id2 = w*get_global_size(1) + get_global_id(1); + ytvalues[w] = LoadVector(id2, n, ygm, y_offset, y_inc, !is_rowmajor); + } + + // Sets the proper value of alpha in case conjugation is needed + real alpha1 = alpha; + real alpha2 = alpha; + #if defined(ROUTINE_HER2) || defined(ROUTINE_HPR2) + if (is_rowmajor) { + COMPLEX_CONJUGATE(alpha1); + } + else { + COMPLEX_CONJUGATE(alpha2); + } + #endif + + // Loops over the work per thread twice + #pragma unroll + for (int w1=0; w1<WPT; ++w1) { + #pragma unroll + for (int w2=0; w2<WPT; ++w2) { + + // Global thread IDs + const int id1 = w1*get_global_size(0) + get_global_id(0); + const int id2 = w2*get_global_size(1) + get_global_id(1); + + // Skip these threads if they do not contain threads contributing to the matrix-triangle + if ((is_upper && (id1 > id2)) || (!is_upper && (id2 > id1))) { + // Do nothing + } + + // Loads A, performs the operation, and stores the result into A + else { + MatrixUpdate2(id1, id2, n, n, agm, a_offset, a_ld, + alpha1, xvalues[w2], yvalues[w1], + alpha2, xtvalues[w1], ytvalues[w2], is_upper); + } + } + } +} + +// ================================================================================================= + +// End of the C++11 raw string literal +)" + +// ================================================================================================= diff --git a/src/routines/level2/xher2.cc b/src/routines/level2/xher2.cc new file mode 100644 index 00000000..63144f77 --- /dev/null +++ b/src/routines/level2/xher2.cc @@ -0,0 +1,114 @@ + +// ================================================================================================= +// This file is part of the CLBlast project. The project is licensed under Apache Version 2.0. This +// project loosely follows the Google C++ styleguide and uses a tab-size of two spaces and a max- +// width of 100 characters per line. +// +// Author(s): +// Cedric Nugteren <www.cedricnugteren.nl> +// +// This file implements the Xher2 class (see the header for information about the class). +// +// ================================================================================================= + +#include "internal/routines/level2/xher2.h" + +#include <string> + +namespace clblast { +// ================================================================================================= + +// Specific implementations to get the memory-type based on a template argument +template <> const Precision Xher2<float>::precision_ = Precision::kSingle; +template <> const Precision Xher2<double>::precision_ = Precision::kDouble; +template <> const Precision Xher2<float2>::precision_ = Precision::kComplexSingle; +template <> const Precision Xher2<double2>::precision_ = Precision::kComplexDouble; + +// ================================================================================================= + +// Constructor: forwards to base class constructor +template <typename T> +Xher2<T>::Xher2(Queue &queue, Event &event, const std::string &name): + Routine<T>(queue, event, name, {"Xger"}, precision_) { + source_string_ = + #include "../../kernels/level2/level2.opencl" + #include "../../kernels/level2/xher2.opencl" + ; +} + +// ================================================================================================= + +// The main routine +template <typename T> +StatusCode Xher2<T>::DoHer2(const Layout layout, const Triangle triangle, + const size_t n, + const T alpha, + const Buffer<T> &x_buffer, const size_t x_offset, const size_t x_inc, + const Buffer<T> &y_buffer, const size_t y_offset, const size_t y_inc, + const Buffer<T> &a_buffer, const size_t a_offset, const size_t a_ld, + const bool packed) { + + // Makes sure the dimensions are larger than zero + if (n == 0) { return StatusCode::kInvalidDimension; } + + // The data is either in the upper or lower triangle + const auto is_upper = ((triangle == Triangle::kUpper && layout != Layout::kRowMajor) || + (triangle == Triangle::kLower && layout == Layout::kRowMajor)); + const auto is_rowmajor = (layout == Layout::kRowMajor); + + // Tests the matrix and the vectors for validity + auto status = StatusCode::kSuccess; + if (packed) { status = TestMatrixAP(n, a_buffer, a_offset, sizeof(T)); } + else { status = TestMatrixA(n, n, a_buffer, a_offset, a_ld, sizeof(T)); } + if (ErrorIn(status)) { return status; } + status = TestVectorX(n, x_buffer, x_offset, x_inc, sizeof(T)); + if (ErrorIn(status)) { return status; } + status = TestVectorY(n, y_buffer, y_offset, y_inc, sizeof(T)); + if (ErrorIn(status)) { return status; } + + // Retrieves the Xgemv kernel from the compiled binary + try { + auto& program = GetProgramFromCache(); + auto kernel = Kernel(program, "Xher2"); + + // Sets the kernel arguments + kernel.SetArgument(0, static_cast<int>(n)); + kernel.SetArgument(1, alpha); + kernel.SetArgument(2, x_buffer()); + kernel.SetArgument(3, static_cast<int>(x_offset)); + kernel.SetArgument(4, static_cast<int>(x_inc)); + kernel.SetArgument(5, y_buffer()); + kernel.SetArgument(6, static_cast<int>(y_offset)); + kernel.SetArgument(7, static_cast<int>(y_inc)); + kernel.SetArgument(8, a_buffer()); + kernel.SetArgument(9, static_cast<int>(a_offset)); + kernel.SetArgument(10, static_cast<int>(a_ld)); + kernel.SetArgument(11, static_cast<int>(is_upper)); + kernel.SetArgument(12, static_cast<int>(is_rowmajor)); + + // Launches the kernel + auto global_one = CeilDiv(Ceil(n, db_["WGS1"]), db_["WPT"]); + auto global_two = CeilDiv(Ceil(n, db_["WGS2"]), db_["WPT"]); + auto global = std::vector<size_t>{global_one, global_two}; + auto local = std::vector<size_t>{db_["WGS1"], db_["WGS2"]}; + status = RunKernel(kernel, global, local); + if (ErrorIn(status)) { return status; } + + // Waits for all kernels to finish + queue_.Finish(); + + // Succesfully finished the computation + return StatusCode::kSuccess; + } catch (...) { return StatusCode::kInvalidKernel; } +} + +// ================================================================================================= + +// Compiles the templated class +template class Xher2<float>; +template class Xher2<double>; +template class Xher2<float2>; +template class Xher2<double2>; + +// ================================================================================================= +} // namespace clblast diff --git a/src/routines/level2/xhpr.cc b/src/routines/level2/xhpr.cc index abe00669..24d7ae95 100644 --- a/src/routines/level2/xhpr.cc +++ b/src/routines/level2/xhpr.cc @@ -34,7 +34,7 @@ StatusCode Xhpr<T,U>::DoHpr(const Layout layout, const Triangle triangle, const Buffer<T> &x_buffer, const size_t x_offset, const size_t x_inc, const Buffer<T> &ap_buffer, const size_t ap_offset) { - // + // Specific Xhpr functionality is implemented in the kernel using defines return DoHer(layout, triangle, n, alpha, x_buffer, x_offset, x_inc, ap_buffer, ap_offset, n, true); diff --git a/src/routines/level2/xspr.cc b/src/routines/level2/xspr.cc index 5159ad50..7ef41fba 100644 --- a/src/routines/level2/xspr.cc +++ b/src/routines/level2/xspr.cc @@ -34,7 +34,7 @@ StatusCode Xspr<T>::DoSpr(const Layout layout, const Triangle triangle, const Buffer<T> &x_buffer, const size_t x_offset, const size_t x_inc, const Buffer<T> &ap_buffer, const size_t ap_offset) { - // + // Specific Xspr functionality is implemented in the kernel using defines return DoHer(layout, triangle, n, alpha, x_buffer, x_offset, x_inc, ap_buffer, ap_offset, n, true); diff --git a/src/routines/level2/xsyr.cc b/src/routines/level2/xsyr.cc index 755fde0d..c01fa2d3 100644 --- a/src/routines/level2/xsyr.cc +++ b/src/routines/level2/xsyr.cc @@ -34,7 +34,7 @@ StatusCode Xsyr<T>::DoSyr(const Layout layout, const Triangle triangle, const Buffer<T> &x_buffer, const size_t x_offset, const size_t x_inc, const Buffer<T> &a_buffer, const size_t a_offset, const size_t a_ld) { - // + // Specific Xsyr functionality is implemented in the kernel using defines return DoHer(layout, triangle, n, alpha, x_buffer, x_offset, x_inc, a_buffer, a_offset, a_ld); diff --git a/src/routines/level2/xsyr2.cc b/src/routines/level2/xsyr2.cc new file mode 100644 index 00000000..6db55085 --- /dev/null +++ b/src/routines/level2/xsyr2.cc @@ -0,0 +1,52 @@ + +// ================================================================================================= +// This file is part of the CLBlast project. The project is licensed under Apache Version 2.0. This +// project loosely follows the Google C++ styleguide and uses a tab-size of two spaces and a max- +// width of 100 characters per line. +// +// Author(s): +// Cedric Nugteren <www.cedricnugteren.nl> +// +// This file implements the Xsyr2 class (see the header for information about the class). +// +// ================================================================================================= + +#include "internal/routines/level2/xsyr2.h" + +#include <string> + +namespace clblast { +// ================================================================================================= + +// Constructor: forwards to base class constructor +template <typename T> +Xsyr2<T>::Xsyr2(Queue &queue, Event &event, const std::string &name): + Xher2<T>(queue, event, name) { +} + +// ================================================================================================= + +// The main routine +template <typename T> +StatusCode Xsyr2<T>::DoSyr2(const Layout layout, const Triangle triangle, + const size_t n, + const T alpha, + const Buffer<T> &x_buffer, const size_t x_offset, const size_t x_inc, + const Buffer<T> &y_buffer, const size_t y_offset, const size_t y_inc, + const Buffer<T> &a_buffer, const size_t a_offset, const size_t a_ld) { + + // Specific Xsyr2 functionality is implemented in the kernel using defines + return DoHer2(layout, triangle, n, alpha, + x_buffer, x_offset, x_inc, + y_buffer, y_offset, y_inc, + a_buffer, a_offset, a_ld); +} + +// ================================================================================================= + +// Compiles the templated class +template class Xsyr2<float>; +template class Xsyr2<double>; + +// ================================================================================================= +} // namespace clblast |