Merge branch 'master' into CLBlast-267-convgemm

6 files changed, 77 insertions, 52 deletions

diff --git a/src/routines/common.cpp b/src/routines/common.cpp
index 5b80e3f2..695785c4 100644
--- a/src/routines/common.cpp
+++ b/src/routines/common.cpp
@@ -13,6 +13,7 @@
 
 #include <vector>
 #include <chrono>
+#include <iostream>
 
 #include "routines/common.hpp"
 
@@ -38,13 +39,22 @@ void RunKernel(Kernel &kernel, Queue &queue, const Device &device,
     auto local_size = size_t{1};
     for (auto &item: local) { local_size *= item; }
     if (local_size > device.MaxWorkGroupSize()) {
-      throw RuntimeErrorCode(StatusCode::kInvalidLocalThreadsTotal);
+      throw RuntimeErrorCode(StatusCode::kInvalidLocalThreadsTotal,
+                             ToString(local_size) + " is larger than " + ToString(device.MaxWorkGroupSize()));
     }
 
     // Make sure the global thread sizes are at least equal to the local sizes
     for (auto i=size_t{0}; i<global.size(); ++i) {
       if (global[i] < local[i]) { global[i] = local[i]; }
     }
+
+    // Verify that the global thread sizes are a multiple of the local sizes
+    for (auto i=size_t{0}; i<global.size(); ++i) {
+      if ((global[i] / local[i]) * local[i] != global[i]) {
+        throw RuntimeErrorCode(StatusCode::kInvalidLocalThreadsDim,
+                               ToString(global[i]) + " is not divisible by " + ToString(local[i]));
+      }
+    }
   }
 
   // Tests for local memory usage
@@ -77,11 +87,10 @@ void RunKernel(Kernel &kernel, Queue &queue, const Device &device,
 // Sets all elements of a matrix to a constant value
 template <typename T>
 void FillMatrix(Queue &queue, const Device &device,
-                const std::shared_ptr<Program> program, const Databases &,
+                const std::shared_ptr<Program> program,
                 EventPointer event, const std::vector<Event> &waitForEvents,
                 const size_t m, const size_t n, const size_t ld, const size_t offset,
-                const Buffer<T> &dest,
-                const T constant_value) {
+                const Buffer<T> &dest, const T constant_value, const size_t local_size) {
   auto kernel = Kernel(program, "FillMatrix");
   kernel.SetArgument(0, static_cast<int>(m));
   kernel.SetArgument(1, static_cast<int>(n));
@@ -89,63 +98,62 @@ void FillMatrix(Queue &queue, const Device &device,
   kernel.SetArgument(3, static_cast<int>(offset));
   kernel.SetArgument(4, dest());
   kernel.SetArgument(5, GetRealArg(constant_value));
-  auto local = std::vector<size_t>{16, 1};
-  auto global = std::vector<size_t>{Ceil(m, 16), n};
+  auto local = std::vector<size_t>{local_size, 1};
+  auto global = std::vector<size_t>{Ceil(m, local_size), n};
   RunKernel(kernel, queue, device, global, local, event, waitForEvents);
 }
 
 // Compiles the above function
-template void FillMatrix<half>(Queue&, const Device&, const std::shared_ptr<Program>, const Databases&,
+template void FillMatrix<half>(Queue&, const Device&, const std::shared_ptr<Program>,
                                EventPointer, const std::vector<Event>&, const size_t, const size_t,
-                               const size_t, const size_t, const Buffer<half>&, const half);
-template void FillMatrix<float>(Queue&, const Device&, const std::shared_ptr<Program>, const Databases&,
+                               const size_t, const size_t, const Buffer<half>&, const half, const size_t);
+template void FillMatrix<float>(Queue&, const Device&, const std::shared_ptr<Program>,
                                 EventPointer, const std::vector<Event>&, const size_t, const size_t,
-                                const size_t, const size_t, const Buffer<float>&, const float);
-template void FillMatrix<double>(Queue&, const Device&, const std::shared_ptr<Program>, const Databases&,
+                                const size_t, const size_t, const Buffer<float>&, const float, const size_t);
+template void FillMatrix<double>(Queue&, const Device&, const std::shared_ptr<Program>,
                                  EventPointer, const std::vector<Event>&, const size_t, const size_t,
-                                 const size_t, const size_t, const Buffer<double>&, const double);
-template void FillMatrix<float2>(Queue&, const Device&, const std::shared_ptr<Program>, const Databases&,
+                                 const size_t, const size_t, const Buffer<double>&, const double, const size_t);
+template void FillMatrix<float2>(Queue&, const Device&, const std::shared_ptr<Program>,
                                  EventPointer, const std::vector<Event>&, const size_t, const size_t,
-                                 const size_t, const size_t, const Buffer<float2>&, const float2);
-template void FillMatrix<double2>(Queue&, const Device&, const std::shared_ptr<Program>, const Databases&,
+                                 const size_t, const size_t, const Buffer<float2>&, const float2, const size_t);
+template void FillMatrix<double2>(Queue&, const Device&, const std::shared_ptr<Program>,
                                   EventPointer, const std::vector<Event>&, const size_t, const size_t,
-                                  const size_t, const size_t, const Buffer<double2>&, const double2);
+                                  const size_t, const size_t, const Buffer<double2>&, const double2, const size_t);
 
 // Sets all elements of a vector to a constant value
 template <typename T>
 void FillVector(Queue &queue, const Device &device,
-                const std::shared_ptr<Program> program, const Databases &,
+                const std::shared_ptr<Program> program,
                 EventPointer event, const std::vector<Event> &waitForEvents,
                 const size_t n, const size_t inc, const size_t offset,
-                const Buffer<T> &dest,
-                const T constant_value) {
+                const Buffer<T> &dest, const T constant_value, const size_t local_size) {
   auto kernel = Kernel(program, "FillVector");
   kernel.SetArgument(0, static_cast<int>(n));
   kernel.SetArgument(1, static_cast<int>(inc));
   kernel.SetArgument(2, static_cast<int>(offset));
   kernel.SetArgument(3, dest());
   kernel.SetArgument(4, GetRealArg(constant_value));
-  auto local = std::vector<size_t>{16};
-  auto global = std::vector<size_t>{Ceil(n, 16)};
+  auto local = std::vector<size_t>{local_size};
+  auto global = std::vector<size_t>{Ceil(n, local_size)};
   RunKernel(kernel, queue, device, global, local, event, waitForEvents);
 }
 
 // Compiles the above function
-template void FillVector<half>(Queue&, const Device&, const std::shared_ptr<Program>, const Databases&,
+template void FillVector<half>(Queue&, const Device&, const std::shared_ptr<Program>,
                                EventPointer, const std::vector<Event>&, const size_t, const size_t,
-                               const size_t, const Buffer<half>&, const half);
-template void FillVector<float>(Queue&, const Device&, const std::shared_ptr<Program>, const Databases&,
+                               const size_t, const Buffer<half>&, const half, const size_t);
+template void FillVector<float>(Queue&, const Device&, const std::shared_ptr<Program>,
                                 EventPointer, const std::vector<Event>&, const size_t, const size_t,
-                                const size_t, const Buffer<float>&, const float);
-template void FillVector<double>(Queue&, const Device&, const std::shared_ptr<Program>, const Databases&,
+                                const size_t, const Buffer<float>&, const float, const size_t);
+template void FillVector<double>(Queue&, const Device&, const std::shared_ptr<Program>,
                                  EventPointer, const std::vector<Event>&, const size_t, const size_t,
-                                 const size_t, const Buffer<double>&, const double);
-template void FillVector<float2>(Queue&, const Device&, const std::shared_ptr<Program>, const Databases&,
+                                 const size_t, const Buffer<double>&, const double, const size_t);
+template void FillVector<float2>(Queue&, const Device&, const std::shared_ptr<Program>,
                                  EventPointer, const std::vector<Event>&, const size_t, const size_t,
-                                 const size_t, const Buffer<float2>&, const float2);
-template void FillVector<double2>(Queue&, const Device&, const std::shared_ptr<Program>, const Databases&,
+                                 const size_t, const Buffer<float2>&, const float2, const size_t);
+template void FillVector<double2>(Queue&, const Device&, const std::shared_ptr<Program>,
                                   EventPointer, const std::vector<Event>&, const size_t, const size_t,
-                                  const size_t, const Buffer<double2>&, const double2);
+                                  const size_t, const Buffer<double2>&, const double2, const size_t);
 
 // =================================================================================================
 } // namespace clblast
diff --git a/src/routines/common.hpp b/src/routines/common.hpp
index b909243d..c30a2e0e 100644
--- a/src/routines/common.hpp
+++ b/src/routines/common.hpp
@@ -36,20 +36,18 @@ void RunKernel(Kernel &kernel, Queue &queue, const Device &device,
 // Sets all elements of a matrix to a constant value
 template <typename T>
 void FillMatrix(Queue &queue, const Device &device,
-                const std::shared_ptr<Program> program, const Databases &,
+                const std::shared_ptr<Program> program,
                 EventPointer event, const std::vector<Event> &waitForEvents,
                 const size_t m, const size_t n, const size_t ld, const size_t offset,
-                const Buffer<T> &dest,
-                const T constant_value);
+                const Buffer<T> &dest, const T constant_value, const size_t local_size);
 
 // Sets all elements of a vector to a constant value
 template <typename T>
 void FillVector(Queue &queue, const Device &device,
-                const std::shared_ptr<Program> program, const Databases &,
+                const std::shared_ptr<Program> program,
                 EventPointer event, const std::vector<Event> &waitForEvents,
                 const size_t n, const size_t inc, const size_t offset,
-                const Buffer<T> &dest,
-                const T constant_value);
+                const Buffer<T> &dest, const T constant_value, const size_t local_size);
 
 // =================================================================================================
 
diff --git a/src/routines/level2/xtrsv.cpp b/src/routines/level2/xtrsv.cpp
index 36c33a76..76401753 100644
--- a/src/routines/level2/xtrsv.cpp
+++ b/src/routines/level2/xtrsv.cpp
@@ -68,7 +68,7 @@ void Xtrsv<T>::Substitution(const Layout layout, const Triangle triangle,
 
   // Launches the kernel
   const auto local = std::vector<size_t>{db_["TRSV_BLOCK_SIZE"]};
-  const auto global = std::vector<size_t>{1};
+  const auto global = std::vector<size_t>{Ceil(n, db_["TRSV_BLOCK_SIZE"])};
   auto event = Event();
   RunKernel(kernel, queue_, device_, global, local, event.pointer());
   event.WaitForCompletion();
@@ -87,6 +87,11 @@ void Xtrsv<T>::DoTrsv(const Layout layout, const Triangle triangle,
   // Makes sure all dimensions are larger than zero
   if (n == 0) { throw BLASError(StatusCode::kInvalidDimension); }
 
+  // Some parts of this kernel are not tunable and thus require some minimal OpenCL properties
+  if (device_.MaxWorkGroupSize() < 16) { // minimum of total local work size of 16
+    throw RuntimeErrorCode(StatusCode::kNotImplemented);
+  }
+
   // Tests the matrix and vector
   TestMatrixA(n, n, a_buffer, a_offset, a_ld);
   TestVectorX(n, b_buffer, b_offset, b_inc);
@@ -102,8 +107,8 @@ void Xtrsv<T>::DoTrsv(const Layout layout, const Triangle triangle,
   // Fills the output buffer with zeros
   auto eventWaitList = std::vector<Event>();
   auto fill_vector_event = Event();
-  FillVector(queue_, device_, program_, db_, fill_vector_event.pointer(), eventWaitList,
-             n, x_inc, x_offset, x_buffer, ConstantZero<T>());
+  FillVector(queue_, device_, program_, fill_vector_event.pointer(), eventWaitList,
+             n, x_inc, x_offset, x_buffer, ConstantZero<T>(), 16);
   fill_vector_event.WaitForCompletion();
 
   // Derives properties based on the arguments
diff --git a/src/routines/level3/xgemm.hpp b/src/routines/level3/xgemm.hpp
index ec84fbb7..ed8cc69d 100644
--- a/src/routines/level3/xgemm.hpp
+++ b/src/routines/level3/xgemm.hpp
@@ -25,9 +25,9 @@ class Xgemm: public Routine {
  public:
 
   // Defines the assumptions of the GEMM kernels
-  static const bool a_want_rotated_(const size_t gemm_kernel_id) { return gemm_kernel_id == 1; }
-  static const bool b_want_rotated_(const size_t gemm_kernel_id) { return true; }
-  static const bool c_want_rotated_(const size_t gemm_kernel_id) { return gemm_kernel_id == 1; }
+  static bool a_want_rotated_(const size_t gemm_kernel_id) { return gemm_kernel_id == 1; }
+  static bool b_want_rotated_(const size_t) { return true; }
+  static bool c_want_rotated_(const size_t gemm_kernel_id) { return gemm_kernel_id == 1; }
 
   // Computes the size of the temporary GEMM buffer based on user-arguments
   static size_t GetTempSize(const Layout layout, const Transpose a_transpose, const Transpose b_transpose,
diff --git a/src/routines/level3/xtrsm.cpp b/src/routines/level3/xtrsm.cpp
index d622e3bf..905660ff 100644
--- a/src/routines/level3/xtrsm.cpp
+++ b/src/routines/level3/xtrsm.cpp
@@ -78,6 +78,11 @@ void Xtrsm<T>::TrsmColMajor(const Side side, const Triangle triangle,
   // Makes sure all dimensions are larger than zero
   if ((m == 0) || (n == 0)) { throw BLASError(StatusCode::kInvalidDimension); }
 
+  // Some parts of this kernel are not tunable and thus require some minimal OpenCL properties
+  if (device_.MaxWorkGroupSize() < 16) { // minimum of total local work size of 16
+    throw RuntimeErrorCode(StatusCode::kNotImplemented);
+  }
+
   // Computes the k dimension. This is based on whether or not matrix is A (on the left)
   // or B (on the right) in the Xgemm routine.
   const auto k = (side == Side::kLeft) ? m : n;
@@ -105,8 +110,8 @@ void Xtrsm<T>::TrsmColMajor(const Side side, const Triangle triangle,
   // Fills the output buffer with zeros
   auto eventWaitList = std::vector<Event>();
   auto fill_matrix_event = Event();
-  FillMatrix(queue_, device_, program_, db_, fill_matrix_event.pointer(), eventWaitList,
-             x_one, x_two, x_ld, x_offset, x_buffer, ConstantZero<T>());
+  FillMatrix(queue_, device_, program_, fill_matrix_event.pointer(), eventWaitList,
+             x_one, x_two, x_ld, x_offset, x_buffer, ConstantZero<T>(), 16);
   fill_matrix_event.WaitForCompletion();
 
   // Inverts the diagonal blocks
diff --git a/src/routines/levelx/xinvert.cpp b/src/routines/levelx/xinvert.cpp
index a5ef9e10..eea8527a 100644
--- a/src/routines/levelx/xinvert.cpp
+++ b/src/routines/levelx/xinvert.cpp
@@ -49,9 +49,16 @@ void Xinvert<T>::InvertMatrixDiagonalBlocks(const Layout layout, const Triangle
     throw BLASError(StatusCode::kInvalidDimension);
   }
 
+  // Some parts of this kernel are not tunable and thus require some minimal OpenCL properties
+  if (device_.MaxWorkGroupSize() < 16) { // minimum of total local work size of 16
+    throw RuntimeErrorCode(StatusCode::kNotImplemented);
+  }
+
   // Helper variables
   const auto internal_block_size = static_cast<size_t>(db_["INTERNAL_BLOCK_SIZE"]);
-  assert(internal_block_size == 16);
+  if (internal_block_size != 16) {
+    throw RuntimeErrorCode(StatusCode::kNotImplemented); // e.g. Apple CPU OpenCL with a WGS of 1
+  }                                                      // when barriers are present
   const auto num_blocks = CeilDiv(n, block_size);
   const auto num_internal_blocks = CeilDiv(n, internal_block_size);
   const auto unit_diagonal = (diag == Diagonal::kUnit) ? true : false;
@@ -75,8 +82,9 @@ void Xinvert<T>::InvertMatrixDiagonalBlocks(const Layout layout, const Triangle
   // Fills the output buffer with zeros
   auto event_wait_list = std::vector<Event>();
   auto fill_matrix_event = Event();
-  FillMatrix(queue_, device_, program_, db_, fill_matrix_event.pointer(), event_wait_list,
-             block_size, num_blocks * block_size, block_size, 0, dest, ConstantZero<T>());
+  FillMatrix(queue_, device_, program_, fill_matrix_event.pointer(), event_wait_list,
+             block_size, num_blocks * block_size, block_size, 0, dest, ConstantZero<T>(),
+             16);
   event_wait_list.push_back(fill_matrix_event);
 
   // Inverts the diagonal IB by IB inner blocks of the matrix: one block per work-group
@@ -89,11 +97,11 @@ void Xinvert<T>::InvertMatrixDiagonalBlocks(const Layout layout, const Triangle
   kernel.SetArgument(5, static_cast<int>(block_size));
   kernel.SetArgument(6, static_cast<int>(unit_diagonal));
   kernel.SetArgument(7, static_cast<int>(is_upper));
-  const auto local = std::vector<size_t>{internal_block_size};
-  const auto global = std::vector<size_t>{num_internal_blocks * internal_block_size};
+  const auto local_invert = std::vector<size_t>{internal_block_size};
+  const auto global_invert = std::vector<size_t>{num_internal_blocks * internal_block_size};
   auto base_kernel_event = Event();
   auto base_kernel_event_pointer = (internal_block_size == block_size) ? event_ : base_kernel_event.pointer();
-  RunKernel(kernel, queue_, device_, global, local, base_kernel_event_pointer, event_wait_list);
+  RunKernel(kernel, queue_, device_, global_invert, local_invert, base_kernel_event_pointer, event_wait_list);
   if (internal_block_size == block_size) { event_wait_list.push_back(base_kernel_event); }
 
   // Builds up block_size x block_size blocks. For example, internal_block_size=16:
@@ -107,7 +115,8 @@ void Xinvert<T>::InvertMatrixDiagonalBlocks(const Layout layout, const Triangle
     const auto npages = CeilDiv(n, current_size*2);
     const auto local0 = (current_size <= 32) ? current_size/4 : 16;
     const auto local = std::vector<size_t>{local0, 4};
-    const auto global = std::vector<size_t>{(current_size/local[1]), npages*(current_size/16)*local[1]};
+    const auto global = std::vector<size_t>{Ceil(current_size/local[1], local[0]),
+                                            Ceil(npages*(current_size/16)*local[1], local[1])};
 
     // Part 1
     auto kernel1 = Kernel(program_, "TripleMatMul" + ToString(current_size) + "Part1" + name_postfix);