diff options
Diffstat (limited to 'src')
-rw-r--r-- | src/clblast.cpp | 84 | ||||
-rw-r--r-- | src/clblast_c.cpp | 157 | ||||
-rw-r--r-- | src/routines/levelx/xgemmbatched.cpp | 115 | ||||
-rw-r--r-- | src/routines/levelx/xgemmbatched.hpp | 47 |
4 files changed, 403 insertions, 0 deletions
diff --git a/src/clblast.cpp b/src/clblast.cpp index d3db8edf..a8bcf91d 100644 --- a/src/clblast.cpp +++ b/src/clblast.cpp @@ -72,6 +72,7 @@ // Level-x includes (non-BLAS) #include "routines/levelx/xomatcopy.hpp" #include "routines/levelx/xaxpybatched.hpp" +#include "routines/levelx/xgemmbatched.hpp" namespace clblast { @@ -2231,6 +2232,89 @@ template StatusCode PUBLIC_API AxpyBatched<half>(const size_t, cl_mem, const size_t*, const size_t, const size_t, cl_command_queue*, cl_event*); + +// Batched version of GEMM: SGEMMBATCHED/DGEMMBATCHED/CGEMMBATCHED/ZGEMMBATCHED/HGEMMBATCHED +template <typename T> +StatusCode GemmBatched(const Layout layout, const Transpose a_transpose, const Transpose b_transpose, + const size_t m, const size_t n, const size_t k, + const T *alphas, + const cl_mem a_buffer, const size_t *a_offsets, const size_t a_ld, + const cl_mem b_buffer, const size_t *b_offsets, const size_t b_ld, + const T *betas, + cl_mem c_buffer, const size_t *c_offsets, const size_t c_ld, + const size_t batch_count, + cl_command_queue* queue, cl_event* event) { + try { + auto queue_cpp = Queue(*queue); + auto routine = XgemmBatched<T>(queue_cpp, event); + auto alphas_cpp = std::vector<T>(); + auto betas_cpp = std::vector<T>(); + auto a_offsets_cpp = std::vector<size_t>(); + auto b_offsets_cpp = std::vector<size_t>(); + auto c_offsets_cpp = std::vector<size_t>(); + for (auto batch = size_t{0}; batch < batch_count; ++batch) { + alphas_cpp.push_back(alphas[batch]); + betas_cpp.push_back(betas[batch]); + a_offsets_cpp.push_back(a_offsets[batch]); + b_offsets_cpp.push_back(b_offsets[batch]); + c_offsets_cpp.push_back(c_offsets[batch]); + } + routine.DoGemmBatched(layout, a_transpose, b_transpose, + m, n, k, + alphas_cpp, + Buffer<T>(a_buffer), a_offsets_cpp, a_ld, + Buffer<T>(b_buffer), b_offsets_cpp, b_ld, + betas_cpp, + Buffer<T>(c_buffer), c_offsets_cpp, c_ld, + batch_count); + return StatusCode::kSuccess; + } catch (...) { return DispatchException(); } +} +template StatusCode PUBLIC_API GemmBatched<float>(const Layout, const Transpose, const Transpose, + const size_t, const size_t, const size_t, + const float*, + const cl_mem, const size_t*, const size_t, + const cl_mem, const size_t*, const size_t, + const float*, + cl_mem, const size_t*, const size_t, + const size_t, + cl_command_queue*, cl_event*); +template StatusCode PUBLIC_API GemmBatched<double>(const Layout, const Transpose, const Transpose, + const size_t, const size_t, const size_t, + const double*, + const cl_mem, const size_t*, const size_t, + const cl_mem, const size_t*, const size_t, + const double*, + cl_mem, const size_t*, const size_t, + const size_t, + cl_command_queue*, cl_event*); +template StatusCode PUBLIC_API GemmBatched<float2>(const Layout, const Transpose, const Transpose, + const size_t, const size_t, const size_t, + const float2*, + const cl_mem, const size_t*, const size_t, + const cl_mem, const size_t*, const size_t, + const float2*, + cl_mem, const size_t*, const size_t, + const size_t, + cl_command_queue*, cl_event*); +template StatusCode PUBLIC_API GemmBatched<double2>(const Layout, const Transpose, const Transpose, + const size_t, const size_t, const size_t, + const double2*, + const cl_mem, const size_t*, const size_t, + const cl_mem, const size_t*, const size_t, + const double2*, + cl_mem, const size_t*, const size_t, + const size_t, + cl_command_queue*, cl_event*); +template StatusCode PUBLIC_API GemmBatched<half>(const Layout, const Transpose, const Transpose, + const size_t, const size_t, const size_t, + const half*, + const cl_mem, const size_t*, const size_t, + const cl_mem, const size_t*, const size_t, + const half*, + cl_mem, const size_t*, const size_t, + const size_t, + cl_command_queue*, cl_event*); // ================================================================================================= // Clears the cache of stored binaries diff --git a/src/clblast_c.cpp b/src/clblast_c.cpp index b09f8c54..f5104bad 100644 --- a/src/clblast_c.cpp +++ b/src/clblast_c.cpp @@ -3554,6 +3554,163 @@ CLBlastStatusCode CLBlastHaxpyBatched(const size_t n, } catch (...) { return static_cast<CLBlastStatusCode>(clblast::DispatchExceptionForC()); } } +// GEMM +CLBlastStatusCode CLBlastSgemmBatched(const CLBlastLayout layout, const CLBlastTranspose a_transpose, const CLBlastTranspose b_transpose, + const size_t m, const size_t n, const size_t k, + const float *alphas, + const cl_mem a_buffer, const size_t *a_offsets, const size_t a_ld, + const cl_mem b_buffer, const size_t *b_offsets, const size_t b_ld, + const float *betas, + cl_mem c_buffer, const size_t *c_offsets, const size_t c_ld, + const size_t batch_count, + cl_command_queue* queue, cl_event* event) { + auto alphas_cpp = std::vector<float>(); + auto betas_cpp = std::vector<float>(); + for (auto batch = size_t{0}; batch < batch_count; ++batch) { + alphas_cpp.push_back(alphas[batch]); + betas_cpp.push_back(betas[batch]); + } + try { + return static_cast<CLBlastStatusCode>( + clblast::GemmBatched(static_cast<clblast::Layout>(layout), + static_cast<clblast::Transpose>(a_transpose), + static_cast<clblast::Transpose>(b_transpose), + m, n, k, + alphas_cpp.data(), + a_buffer, a_offsets, a_ld, + b_buffer, b_offsets, b_ld, + betas_cpp.data(), + c_buffer, c_offsets, c_ld, + batch_count, + queue, event) + ); + } catch (...) { return static_cast<CLBlastStatusCode>(clblast::DispatchExceptionForC()); } +} +CLBlastStatusCode CLBlastDgemmBatched(const CLBlastLayout layout, const CLBlastTranspose a_transpose, const CLBlastTranspose b_transpose, + const size_t m, const size_t n, const size_t k, + const double *alphas, + const cl_mem a_buffer, const size_t *a_offsets, const size_t a_ld, + const cl_mem b_buffer, const size_t *b_offsets, const size_t b_ld, + const double *betas, + cl_mem c_buffer, const size_t *c_offsets, const size_t c_ld, + const size_t batch_count, + cl_command_queue* queue, cl_event* event) { + auto alphas_cpp = std::vector<double>(); + auto betas_cpp = std::vector<double>(); + for (auto batch = size_t{0}; batch < batch_count; ++batch) { + alphas_cpp.push_back(alphas[batch]); + betas_cpp.push_back(betas[batch]); + } + try { + return static_cast<CLBlastStatusCode>( + clblast::GemmBatched(static_cast<clblast::Layout>(layout), + static_cast<clblast::Transpose>(a_transpose), + static_cast<clblast::Transpose>(b_transpose), + m, n, k, + alphas_cpp.data(), + a_buffer, a_offsets, a_ld, + b_buffer, b_offsets, b_ld, + betas_cpp.data(), + c_buffer, c_offsets, c_ld, + batch_count, + queue, event) + ); + } catch (...) { return static_cast<CLBlastStatusCode>(clblast::DispatchExceptionForC()); } +} +CLBlastStatusCode CLBlastCgemmBatched(const CLBlastLayout layout, const CLBlastTranspose a_transpose, const CLBlastTranspose b_transpose, + const size_t m, const size_t n, const size_t k, + const cl_float2 *alphas, + const cl_mem a_buffer, const size_t *a_offsets, const size_t a_ld, + const cl_mem b_buffer, const size_t *b_offsets, const size_t b_ld, + const cl_float2 *betas, + cl_mem c_buffer, const size_t *c_offsets, const size_t c_ld, + const size_t batch_count, + cl_command_queue* queue, cl_event* event) { + auto alphas_cpp = std::vector<float2>(); + auto betas_cpp = std::vector<float2>(); + for (auto batch = size_t{0}; batch < batch_count; ++batch) { + alphas_cpp.push_back(float2{alphas[batch].s[0], alphas[batch].s[1]}); + betas_cpp.push_back(float2{betas[batch].s[0], betas[batch].s[1]}); + } + try { + return static_cast<CLBlastStatusCode>( + clblast::GemmBatched(static_cast<clblast::Layout>(layout), + static_cast<clblast::Transpose>(a_transpose), + static_cast<clblast::Transpose>(b_transpose), + m, n, k, + alphas_cpp.data(), + a_buffer, a_offsets, a_ld, + b_buffer, b_offsets, b_ld, + betas_cpp.data(), + c_buffer, c_offsets, c_ld, + batch_count, + queue, event) + ); + } catch (...) { return static_cast<CLBlastStatusCode>(clblast::DispatchExceptionForC()); } +} +CLBlastStatusCode CLBlastZgemmBatched(const CLBlastLayout layout, const CLBlastTranspose a_transpose, const CLBlastTranspose b_transpose, + const size_t m, const size_t n, const size_t k, + const cl_double2 *alphas, + const cl_mem a_buffer, const size_t *a_offsets, const size_t a_ld, + const cl_mem b_buffer, const size_t *b_offsets, const size_t b_ld, + const cl_double2 *betas, + cl_mem c_buffer, const size_t *c_offsets, const size_t c_ld, + const size_t batch_count, + cl_command_queue* queue, cl_event* event) { + auto alphas_cpp = std::vector<double2>(); + auto betas_cpp = std::vector<double2>(); + for (auto batch = size_t{0}; batch < batch_count; ++batch) { + alphas_cpp.push_back(double2{alphas[batch].s[0], alphas[batch].s[1]}); + betas_cpp.push_back(double2{betas[batch].s[0], betas[batch].s[1]}); + } + try { + return static_cast<CLBlastStatusCode>( + clblast::GemmBatched(static_cast<clblast::Layout>(layout), + static_cast<clblast::Transpose>(a_transpose), + static_cast<clblast::Transpose>(b_transpose), + m, n, k, + alphas_cpp.data(), + a_buffer, a_offsets, a_ld, + b_buffer, b_offsets, b_ld, + betas_cpp.data(), + c_buffer, c_offsets, c_ld, + batch_count, + queue, event) + ); + } catch (...) { return static_cast<CLBlastStatusCode>(clblast::DispatchExceptionForC()); } +} +CLBlastStatusCode CLBlastHgemmBatched(const CLBlastLayout layout, const CLBlastTranspose a_transpose, const CLBlastTranspose b_transpose, + const size_t m, const size_t n, const size_t k, + const cl_half *alphas, + const cl_mem a_buffer, const size_t *a_offsets, const size_t a_ld, + const cl_mem b_buffer, const size_t *b_offsets, const size_t b_ld, + const cl_half *betas, + cl_mem c_buffer, const size_t *c_offsets, const size_t c_ld, + const size_t batch_count, + cl_command_queue* queue, cl_event* event) { + auto alphas_cpp = std::vector<half>(); + auto betas_cpp = std::vector<half>(); + for (auto batch = size_t{0}; batch < batch_count; ++batch) { + alphas_cpp.push_back(alphas[batch]); + betas_cpp.push_back(betas[batch]); + } + try { + return static_cast<CLBlastStatusCode>( + clblast::GemmBatched(static_cast<clblast::Layout>(layout), + static_cast<clblast::Transpose>(a_transpose), + static_cast<clblast::Transpose>(b_transpose), + m, n, k, + alphas_cpp.data(), + a_buffer, a_offsets, a_ld, + b_buffer, b_offsets, b_ld, + betas_cpp.data(), + c_buffer, c_offsets, c_ld, + batch_count, + queue, event) + ); + } catch (...) { return static_cast<CLBlastStatusCode>(clblast::DispatchExceptionForC()); } +} + // ================================================================================================= // Clears the cache of stored binaries diff --git a/src/routines/levelx/xgemmbatched.cpp b/src/routines/levelx/xgemmbatched.cpp new file mode 100644 index 00000000..b07425d5 --- /dev/null +++ b/src/routines/levelx/xgemmbatched.cpp @@ -0,0 +1,115 @@ + +// ================================================================================================= +// This file is part of the CLBlast project. The project is licensed under Apache Version 2.0. This +// project loosely follows the Google C++ styleguide and uses a tab-size of two spaces and a max- +// width of 100 characters per line. +// +// Author(s): +// Cedric Nugteren <www.cedricnugteren.nl> +// +// This file implements the XgemmBatched class (see the header for information about the class). +// +// ================================================================================================= + +#include "routines/levelx/xgemmbatched.hpp" + +#include <string> +#include <vector> + +namespace clblast { +// ================================================================================================= + +// Constructor: forwards to base class constructor +template <typename T> +XgemmBatched<T>::XgemmBatched(Queue &queue, EventPointer event, const std::string &name): + Routine(queue, event, name, + {"Copy","Pad","Transpose","Padtranspose","Xgemm","XgemmDirect","KernelSelection"}, + PrecisionValue<T>(), {}, { + #include "../../kernels/level3/level3.opencl" + #include "../../kernels/level3/copy_fast.opencl" + #include "../../kernels/level3/copy_pad.opencl" + #include "../../kernels/level3/transpose_fast.opencl" + #include "../../kernels/level3/transpose_pad.opencl" + #include "../../kernels/level3/convert_symmetric.opencl" + #include "../../kernels/level3/convert_triangular.opencl" + #include "../../kernels/level3/convert_hermitian.opencl" + , // separated in multiple parts to prevent C1091 in MSVC 2013 + #include "../../kernels/level3/xgemm_direct_part1.opencl" + #include "../../kernels/level3/xgemm_direct_part2.opencl" + #include "../../kernels/level3/xgemm_direct_part3.opencl" + , // separated in multiple parts to prevent C1091 in MSVC 2013 + #include "../../kernels/level3/xgemm_part1.opencl" + #include "../../kernels/level3/xgemm_part2.opencl" + #include "../../kernels/level3/xgemm_part3.opencl" + }) { +} + +// ================================================================================================= + +// The main routine +template <typename T> +void XgemmBatched<T>::DoGemmBatched(const Layout layout, const Transpose a_transpose, const Transpose b_transpose, + const size_t m, const size_t n, const size_t k, + const std::vector<T> &alphas, + const Buffer<T> & a_buffer, const std::vector<size_t> &a_offsets, const size_t a_ld, + const Buffer<T> & b_buffer, const std::vector<size_t> &b_offsets, const size_t b_ld, + const std::vector<T> &betas, + const Buffer<T> & c_buffer, const std::vector<size_t> &c_offsets, const size_t c_ld, + const size_t batch_count) { + + // Tests for a valid batch count + if ((batch_count < 1) || (alphas.size() != batch_count) || (betas.size() != batch_count) || + (a_offsets.size() != batch_count) || (b_offsets.size() != batch_count) || (c_offsets.size() != batch_count)) { + throw BLASError(StatusCode::kInvalidBatchCount); + } + + // Makes sure all dimensions are larger than zero + if ((m == 0) || (n == 0) || (k == 0)) { throw BLASError(StatusCode::kInvalidDimension); } + + // Computes whether or not the matrices are transposed in memory. See GEMM routine for details. + const auto a_rotated = (layout == Layout::kColMajor && a_transpose != Transpose::kNo) || + (layout == Layout::kRowMajor && a_transpose == Transpose::kNo); + const auto b_rotated = (layout == Layout::kColMajor && b_transpose != Transpose::kNo) || + (layout == Layout::kRowMajor && b_transpose == Transpose::kNo); + const auto c_rotated = (layout == Layout::kRowMajor); + static const auto a_want_rotated = false; + static const auto b_want_rotated = true; + static const auto c_want_rotated = false; + const auto a_do_transpose = a_rotated != a_want_rotated; + const auto b_do_transpose = b_rotated != b_want_rotated; + const auto c_do_transpose = c_rotated != c_want_rotated; + + // In case of complex data-types, the transpose can also become a conjugate transpose + const auto a_conjugate = (a_transpose == Transpose::kConjugate); + const auto b_conjugate = (b_transpose == Transpose::kConjugate); + + // Computes the first and second dimensions of the 3 matrices taking into account whether the + // matrices are rotated or not + const auto a_one = (a_rotated) ? k : m; + const auto a_two = (a_rotated) ? m : k; + const auto b_one = (b_rotated) ? n : k; + const auto b_two = (b_rotated) ? k : n; + const auto c_one = (c_rotated) ? n : m; + const auto c_two = (c_rotated) ? m : n; + + // Tests the matrices for validity + for (auto batch = size_t{0}; batch < batch_count; ++batch) { + TestMatrixA(a_one, a_two, a_buffer, a_offsets[batch], a_ld); + TestMatrixB(b_one, b_two, b_buffer, b_offsets[batch], b_ld); + TestMatrixC(c_one, c_two, c_buffer, c_offsets[batch], c_ld); + } + + // StatusCode::kNotImplemented; +} + +// ================================================================================================= + +// Compiles the templated class +template class XgemmBatched<half>; +template class XgemmBatched<float>; +template class XgemmBatched<double>; +template class XgemmBatched<float2>; +template class XgemmBatched<double2>; + +// ================================================================================================= +} // namespace clblast diff --git a/src/routines/levelx/xgemmbatched.hpp b/src/routines/levelx/xgemmbatched.hpp new file mode 100644 index 00000000..710011d8 --- /dev/null +++ b/src/routines/levelx/xgemmbatched.hpp @@ -0,0 +1,47 @@ + +// ================================================================================================= +// This file is part of the CLBlast project. The project is licensed under Apache Version 2.0. This +// project loosely follows the Google C++ styleguide and uses a tab-size of two spaces and a max- +// width of 100 characters per line. +// +// Author(s): +// Cedric Nugteren <www.cedricnugteren.nl> +// +// This file implements the XgemmBatched routine. This is a non-blas batched version of GEMM. +// +// ================================================================================================= + +#ifndef CLBLAST_ROUTINES_XGEMMBATCHED_H_ +#define CLBLAST_ROUTINES_XGEMMBATCHED_H_ + +#include <vector> + +#include "routine.hpp" + +namespace clblast { +// ================================================================================================= + +// See comment at top of file for a description of the class +template <typename T> +class XgemmBatched: public Routine { + public: + + // Constructor + XgemmBatched(Queue &queue, EventPointer event, const std::string &name = "GEMMBATCHED"); + + // Templated-precision implementation of the routine + void DoGemmBatched(const Layout layout, const Transpose a_transpose, const Transpose b_transpose, + const size_t m, const size_t n, const size_t k, + const std::vector<T> &alphas, + const Buffer<T> & a_buffer, const std::vector<size_t> &a_offsets, const size_t a_ld, + const Buffer<T> & b_buffer, const std::vector<size_t> &b_offsets, const size_t b_ld, + const std::vector<T> &betas, + const Buffer<T> & c_buffer, const std::vector<size_t> &c_offsets, const size_t c_ld, + const size_t batch_count); +}; + +// ================================================================================================= +} // namespace clblast + +// CLBLAST_ROUTINES_XGEMMBATCHED_H_ +#endif |