Added the TRMM routine, tester, and client

3 files changed, 221 insertions, 15 deletions

diff --git a/src/clblast.cc b/src/clblast.cc
index e3ce4d39..299d0a18 100644
--- a/src/clblast.cc
+++ b/src/clblast.cc
@@ -28,6 +28,7 @@
 #include "internal/routines/xsymm.h"
 #include "internal/routines/xsyrk.h"
 #include "internal/routines/xsyr2k.h"
+#include "internal/routines/xtrmm.h"
 
 namespace clblast {
 // =================================================================================================
@@ -372,7 +373,6 @@ StatusCode Trmm(const Layout layout, const Side side, const Triangle triangle,
                 cl_command_queue* queue, cl_event* event) {
   auto queue_cpp = CommandQueue(*queue);
   auto event_cpp = Event(*event);
-  /*
   auto routine = Xtrmm<T>(queue_cpp, event_cpp);
 
   // Loads the kernel source-code as an include (C++11 raw string literal)
@@ -394,8 +394,6 @@ StatusCode Trmm(const Layout layout, const Side side, const Triangle triangle,
   return routine.DoTrmm(layout, side, triangle, a_transpose, diagonal, m, n, alpha,
                         Buffer(a_buffer), a_offset, a_ld,
                         Buffer(b_buffer), b_offset, b_ld);
-  */
-  return StatusCode::kSuccess;
 }
 template StatusCode Trmm<float>(const Layout, const Side, const Triangle,
                                 const Transpose, const Diagonal,
diff --git a/src/kernels/pad.opencl b/src/kernels/pad.opencl
index cce0c746..8294fab7 100644
--- a/src/kernels/pad.opencl
+++ b/src/kernels/pad.opencl
@@ -135,15 +135,15 @@ __kernel void SymmLowerToSquared(const int src_dim,
       if (id_two < dest_dim && id_one < dest_dim) {
 
         // Loads data from the lower-symmetric matrix
-        real value;
-        SetToZero(value);
+        real result;
+        SetToZero(result);
         if (id_two < src_dim && id_one < src_dim) {
-          if (id_two <= id_one) { value = src[id_two*src_ld + id_one + src_offset]; }
-          else                  { value = src[id_one*src_ld + id_two + src_offset]; }
+          if (id_two <= id_one) { result = src[id_two*src_ld + id_one + src_offset]; }
+          else                  { result = src[id_one*src_ld + id_two + src_offset]; }
         }
 
-        // Stores the value in the destination matrix
-        dest[id_two*dest_ld + id_one + dest_offset] = value;
+        // Stores the result in the destination matrix
+        dest[id_two*dest_ld + id_one + dest_offset] = result;
       }
     }
   }
@@ -168,15 +168,88 @@ __kernel void SymmUpperToSquared(const int src_dim,
       if (id_two < dest_dim && id_one < dest_dim) {
 
         // Loads data from the upper-symmetric matrix
-        real value;
-        SetToZero(value);
+        real result;
+        SetToZero(result);
         if (id_two < src_dim && id_one < src_dim) {
-          if (id_one <= id_two) { value = src[id_two*src_ld + id_one + src_offset]; }
-          else                  { value = src[id_one*src_ld + id_two + src_offset]; }
+          if (id_one <= id_two) { result = src[id_two*src_ld + id_one + src_offset]; }
+          else                  { result = src[id_one*src_ld + id_two + src_offset]; }
         }
 
-        // Stores the value in the destination matrix
-        dest[id_two*dest_ld + id_one + dest_offset] = value;
+        // Stores the result in the destination matrix
+        dest[id_two*dest_ld + id_one + dest_offset] = result;
+      }
+    }
+  }
+}
+
+// =================================================================================================
+
+// Kernel to populate a squared triangular matrix, given that the triangle which holds the data is
+// stored as the lower-triangle of the input matrix. This uses the padding kernel's parameters.
+__attribute__((reqd_work_group_size(PAD_DIMX, PAD_DIMY, 1)))
+__kernel void TrmmLowerToSquared(const int src_dim,
+                                 const int src_ld, const int src_offset,
+                                 __global const real* restrict src,
+                                 const int dest_dim,
+                                 const int dest_ld, const int dest_offset,
+                                 __global real* dest,
+                                 const int unit_diagonal) {
+
+  // Loops over the work per thread in both dimensions
+  #pragma unroll
+  for (int w_one=0; w_one<PAD_WPTX; ++w_one) {
+    const int id_one = (get_group_id(0)*PAD_WPTX + w_one) * PAD_DIMX + get_local_id(0);
+    #pragma unroll
+    for (int w_two=0; w_two<PAD_WPTY; ++w_two) {
+      const int id_two = (get_group_id(1)*PAD_WPTY + w_two) * PAD_DIMY + get_local_id(1);
+      if (id_two < dest_dim && id_one < dest_dim) {
+
+        // Loads data from the lower-triangular matrix
+        real result;
+        SetToZero(result);
+        if (id_two < src_dim && id_one < src_dim) {
+          if (id_two <= id_one) { result = src[id_two*src_ld + id_one + src_offset]; }
+          if (id_two == id_one && unit_diagonal) { SetToOne(result); }
+          // Else: result is zero
+        }
+
+        // Stores the result in the destination matrix
+        dest[id_two*dest_ld + id_one + dest_offset] = result;
+      }
+    }
+  }
+}
+
+// Same as above, but now the matrix' data is stored in the upper-triangle
+__attribute__((reqd_work_group_size(PAD_DIMX, PAD_DIMY, 1)))
+__kernel void TrmmUpperToSquared(const int src_dim,
+                                 const int src_ld, const int src_offset,
+                                 __global const real* restrict src,
+                                 const int dest_dim,
+                                 const int dest_ld, const int dest_offset,
+                                 __global real* dest,
+                                 const int unit_diagonal) {
+
+  // Loops over the work per thread in both dimensions
+  #pragma unroll
+  for (int w_one=0; w_one<PAD_WPTX; ++w_one) {
+    const int id_one = (get_group_id(0)*PAD_WPTX + w_one) * PAD_DIMX + get_local_id(0);
+    #pragma unroll
+    for (int w_two=0; w_two<PAD_WPTY; ++w_two) {
+      const int id_two = (get_group_id(1)*PAD_WPTY + w_two) * PAD_DIMY + get_local_id(1);
+      if (id_two < dest_dim && id_one < dest_dim) {
+
+        // Loads data from the upper-triangular matrix
+        real result;
+        SetToZero(result);
+        if (id_two < src_dim && id_one < src_dim) {
+          if (id_one <= id_two) { result = src[id_two*src_ld + id_one + src_offset]; }
+          if (id_one == id_two && unit_diagonal) { SetToOne(result); }
+          // Else: result is zero
+        }
+
+        // Stores the result in the destination matrix
+        dest[id_two*dest_ld + id_one + dest_offset] = result;
       }
     }
   }
diff --git a/src/routines/xtrmm.cc b/src/routines/xtrmm.cc
new file mode 100644
index 00000000..543df844
--- /dev/null
+++ b/src/routines/xtrmm.cc
@@ -0,0 +1,135 @@
+
+// =================================================================================================
+// This file is part of the CLBlast project. The project is licensed under Apache Version 2.0. This
+// project loosely follows the Google C++ styleguide and uses a tab-size of two spaces and a max-
+// width of 100 characters per line.
+//
+// Author(s):
+//   Cedric Nugteren <www.cedricnugteren.nl>
+//
+// This file implements the Xtrmm class (see the header for information about the class).
+//
+// =================================================================================================
+
+#include "internal/routines/xtrmm.h"
+
+#include <string>
+#include <vector>
+
+namespace clblast {
+// =================================================================================================
+
+// Constructor: forwards to base class constructor
+template <typename T>
+Xtrmm<T>::Xtrmm(CommandQueue &queue, Event &event):
+    Xgemm<T>(queue, event) {
+}
+
+// =================================================================================================
+
+// The main routine
+template <typename T>
+StatusCode Xtrmm<T>::DoTrmm(const Layout layout, const Side side, const Triangle triangle,
+                            const Transpose a_transpose, const Diagonal diagonal,
+                            const size_t m, const size_t n,
+                            const T alpha,
+                            const Buffer &a_buffer, const size_t a_offset, const size_t a_ld,
+                            const Buffer &b_buffer, const size_t b_offset, const size_t b_ld) {
+
+  // Makes sure all dimensions are larger than zero
+  if ((m == 0) || (n == 0)) { return StatusCode::kInvalidDimension; }
+
+  // Computes the k dimension. This is based on whether or not matrix is A (on the left)
+  // or B (on the right) in the Xgemm routine.
+  auto k = (side == Side::kLeft) ? m : n;
+
+  // Checks for validity of the triangular A matrix
+  auto status = TestMatrixA(k, k, a_buffer, a_offset, a_ld, sizeof(T));
+  if (ErrorIn(status)) { return status; }
+
+  // Determines which kernel to run based on the layout (the Xgemm kernel assumes column-major as
+  // default) and on whether we are dealing with an upper or lower triangle of the triangular matrix
+  bool is_upper = ((triangle == Triangle::kUpper && layout != Layout::kRowMajor) ||
+                   (triangle == Triangle::kLower && layout == Layout::kRowMajor));
+  auto kernel_name = (is_upper) ? "TrmmUpperToSquared" : "TrmmLowerToSquared";
+
+  // Determines whether or not the triangular matrix is unit-diagonal
+  auto unit_diagonal = (diagonal == Diagonal::kUnit) ? true : false;
+
+  // Temporary buffer for a copy of the triangular matrix
+  try {
+    auto temp_triangular = Buffer(context_, CL_MEM_READ_WRITE, k*k*sizeof(T));
+
+    // Creates a general matrix from the triangular matrix to be able to run the regular Xgemm
+    // routine afterwards
+    try {
+      auto& program = GetProgramFromCache();
+      auto kernel = Kernel(program, kernel_name);
+
+      // Sets the arguments for the triangular-to-squared kernel
+      kernel.SetArgument(0, static_cast<int>(k));
+      kernel.SetArgument(1, static_cast<int>(a_ld));
+      kernel.SetArgument(2, static_cast<int>(a_offset));
+      kernel.SetArgument(3, a_buffer());
+      kernel.SetArgument(4, static_cast<int>(k));
+      kernel.SetArgument(5, static_cast<int>(k));
+      kernel.SetArgument(6, static_cast<int>(0));
+      kernel.SetArgument(7, temp_triangular());
+      kernel.SetArgument(8, static_cast<int>(unit_diagonal));
+
+      // Uses the common padding kernel's thread configuration. This is allowed, since the
+      // triangular-to-squared kernel uses the same parameters.
+      auto global = std::vector<size_t>{Ceil(CeilDiv(k, db_["PAD_WPTX"]), db_["PAD_DIMX"]),
+                                        Ceil(CeilDiv(k, db_["PAD_WPTY"]), db_["PAD_DIMY"])};
+      auto local = std::vector<size_t>{db_["PAD_DIMX"], db_["PAD_DIMY"]};
+      status = RunKernel(kernel, global, local);
+      if (ErrorIn(status)) { return status; }
+
+      // Runs the regular Xgemm code with either "B := alpha*A*B" or ...
+      if (side == Side::kLeft) {
+        status = DoGemm(layout, a_transpose, Transpose::kNo,
+                        m, n, k,
+                        alpha,
+                        temp_triangular, 0, k,
+                        b_buffer, b_offset, b_ld,
+                        static_cast<T>(0.0),
+                        b_buffer, b_offset, b_ld);
+      }
+
+      // ... with "B := alpha*B*A". Note that A and B are now reversed.
+      else {
+        status = DoGemm(layout, Transpose::kNo, a_transpose,
+                        m, n, k,
+                        alpha,
+                        b_buffer, b_offset, b_ld,
+                        temp_triangular, 0, k,
+                        static_cast<T>(0.0),
+                        b_buffer, b_offset, b_ld);
+
+        // A and B are now reversed, so also reverse the error codes returned from the Xgemm routine
+        switch(status) {
+          case StatusCode::kInvalidMatrixA:      status = StatusCode::kInvalidMatrixB; break;
+          case StatusCode::kInvalidMatrixB:      status = StatusCode::kInvalidMatrixA; break;
+          case StatusCode::kInvalidLeadDimA:     status = StatusCode::kInvalidLeadDimB; break;
+          case StatusCode::kInvalidLeadDimB:     status = StatusCode::kInvalidLeadDimA; break;
+          case StatusCode::kInsufficientMemoryA: status = StatusCode::kInsufficientMemoryB; break;
+          case StatusCode::kInsufficientMemoryB: status = StatusCode::kInsufficientMemoryA; break;
+        }
+      }
+
+      // Return the status of the Xgemm routine
+      return status;
+    } catch (...) { return StatusCode::kInvalidKernel; }
+  } catch (...) { return StatusCode::kTempBufferAllocFailure; }
+}
+
+// =================================================================================================
+
+// Compiles the templated class
+template class Xtrmm<float>;
+template class Xtrmm<double>;
+template class Xtrmm<float2>;
+template class Xtrmm<double2>;
+
+// =================================================================================================
+} // namespace clblast