Merge branch 'master' into android_support

2 files changed, 92 insertions, 48 deletions

diff --git a/src/routines/level3/xgemm.cpp b/src/routines/level3/xgemm.cpp
index 3909c308..a0063ee2 100644
--- a/src/routines/level3/xgemm.cpp
+++ b/src/routines/level3/xgemm.cpp
@@ -161,10 +161,24 @@ void Xgemm<T>::GemmIndirect(const size_t m, const size_t n, const size_t k,
   auto c_no_temp = c_one == c_one_i && c_two == c_two_i && c_ld == c_one && c_offset == 0 &&
                    c_do_transpose == false;
 
-  // Creates the temporary matrices
-  const auto a_temp = (a_no_temp) ? a_buffer : Buffer<T>(context_, a_one_i*a_two_i);
-  const auto b_temp = (b_no_temp) ? b_buffer : Buffer<T>(context_, b_one_i*b_two_i);
-  const auto c_temp = (c_no_temp) ? c_buffer : Buffer<T>(context_, c_one_i*c_two_i);
+  // Computes the sizes and offsets for (optional) temporary buffers for the 3 matrices
+  auto temp_size = size_t{0};
+  auto b_temp_offset = size_t{0};
+  auto c_temp_offset = size_t{0};
+  if (!a_no_temp) { temp_size += a_one_i*a_two_i; }
+  if (!b_no_temp) { b_temp_offset = temp_size; temp_size += b_one_i*b_two_i; }
+  if (!c_no_temp) { c_temp_offset = temp_size; temp_size += c_one_i*c_two_i; }
+  if (!IsMultiple(b_temp_offset, db_["VWN"])) { throw BLASError(StatusCode::kUnexpectedError); }
+  if (!IsMultiple(c_temp_offset, db_["VWM"])) { throw BLASError(StatusCode::kUnexpectedError); }
+
+  // Creates the buffer for the (optional) temporary matrices. Note that we use 'a_buffer' in case
+  // when no temporary buffer is needed, but that's just to make it compile: it is never used.
+  const auto temp_buffer = (temp_size > 0) ? Buffer<T>(context_, temp_size) : a_buffer;
+
+  // Sets the buffer pointers for (temp) matrices A, B, and C
+  const auto a_temp = (a_no_temp) ? a_buffer : temp_buffer;
+  const auto b_temp = (b_no_temp) ? b_buffer : temp_buffer;
+  const auto c_temp = (c_no_temp) ? c_buffer : temp_buffer;
 
   // Events of all kernels (including pre/post processing kernels)
   auto eventWaitList = std::vector<Event>();
@@ -188,7 +202,7 @@ void Xgemm<T>::GemmIndirect(const size_t m, const size_t n, const size_t k,
     auto eventProcessB = Event();
     PadCopyTransposeMatrix(queue_, device_, db_, eventProcessB.pointer(), emptyEventList,
                            b_one, b_two, b_ld, b_offset, b_buffer,
-                           b_one_i, b_two_i, b_one_i, 0, b_temp,
+                           b_one_i, b_two_i, b_one_i, b_temp_offset, b_temp,
                            ConstantOne<T>(), program_,
                            true, b_do_transpose, b_conjugate);
     eventWaitList.push_back(eventProcessB);
@@ -199,7 +213,7 @@ void Xgemm<T>::GemmIndirect(const size_t m, const size_t n, const size_t k,
     auto eventProcessC = Event();
     PadCopyTransposeMatrix(queue_, device_, db_, eventProcessC.pointer(), emptyEventList,
                            c_one, c_two, c_ld, c_offset, c_buffer,
-                           c_one_i, c_two_i, c_one_i, 0, c_temp,
+                           c_one_i, c_two_i, c_one_i, c_temp_offset, c_temp,
                            ConstantOne<T>(), program_,
                            true, c_do_transpose, false);
     eventWaitList.push_back(eventProcessC);
@@ -217,6 +231,8 @@ void Xgemm<T>::GemmIndirect(const size_t m, const size_t n, const size_t k,
   kernel.SetArgument(5, a_temp());
   kernel.SetArgument(6, b_temp());
   kernel.SetArgument(7, c_temp());
+  kernel.SetArgument(8, static_cast<int>(b_temp_offset / db_["VWN"]));
+  kernel.SetArgument(9, static_cast<int>(c_temp_offset / db_["VWM"]));
 
   // Computes the global and local thread sizes
   const auto global = std::vector<size_t>{
@@ -234,7 +250,7 @@ void Xgemm<T>::GemmIndirect(const size_t m, const size_t n, const size_t k,
   if (!c_no_temp) {
     eventWaitList.push_back(eventKernel);
     PadCopyTransposeMatrix(queue_, device_, db_, event_, eventWaitList,
-                           c_one_i, c_two_i, c_one_i, 0, c_temp,
+                           c_one_i, c_two_i, c_one_i, c_temp_offset, c_temp,
                            c_one, c_two, c_ld, c_offset, c_buffer,
                            ConstantOne<T>(), program_,
                            false, c_do_transpose, false);
diff --git a/src/routines/level3/xtrsm.cpp b/src/routines/level3/xtrsm.cpp
index 685d458b..d622e3bf 100644
--- a/src/routines/level3/xtrsm.cpp
+++ b/src/routines/level3/xtrsm.cpp
@@ -73,7 +73,7 @@ void Xtrsm<T>::TrsmColMajor(const Side side, const Triangle triangle,
                             const Buffer<T> &b_buffer, const size_t b_offset, const size_t b_ld) {
 
   // Settings
-  constexpr auto block_size = size_t{32}; // tuneable
+  constexpr auto block_size = size_t{16}; // tuneable
 
   // Makes sure all dimensions are larger than zero
   if ((m == 0) || (n == 0)) { throw BLASError(StatusCode::kInvalidDimension); }
@@ -128,18 +128,25 @@ void Xtrsm<T>::TrsmColMajor(const Side side, const Triangle triangle,
       for (auto i = size_t{0}; i < m; i += block_size) {
         const auto gemm_alpha = (i == 0) ? alpha : ConstantOne<T>();
         const auto current_block_size = std::min(m - i, block_size);
-        DoGemm(Layout::kColMajor, a_transpose, Transpose::kNo,
-               current_block_size, n, current_block_size, gemm_alpha,
-               a_inv_buffer, i * block_size, block_size,
-               b_buffer, b_offset + i, b_ld, ConstantZero<T>(),
-               x_buffer, x_offset + i, x_ld);
+        auto gemm1_event = Event();
+        auto gemm1 = Xgemm<T>(queue_, gemm1_event.pointer());
+        gemm1.DoGemm(Layout::kColMajor, a_transpose, Transpose::kNo,
+                     current_block_size, n, current_block_size, gemm_alpha,
+                     a_inv_buffer, i * block_size, block_size,
+                     b_buffer, b_offset + i, b_ld, ConstantZero<T>(),
+                     x_buffer, x_offset + i, x_ld);
+        gemm1_event.WaitForCompletion();
         if (i + block_size >= m) { break; }
+
         const auto this_a_offset = (a_transpose == Transpose::kNo) ? (i + block_size) + i * a_ld : i + (block_size + i) * a_ld;
-        DoGemm(Layout::kColMajor, a_transpose, Transpose::kNo,
-               m - i - block_size, n, block_size, ConstantNegOne<T>(),
-               a_buffer, this_a_offset, a_ld,
-               x_buffer, x_offset + i, x_ld, gemm_alpha,
-               b_buffer, b_offset + i + block_size, b_ld);
+        auto gemm2_event = Event();
+        auto gemm2 = Xgemm<T>(queue_, gemm2_event.pointer());
+        gemm2.DoGemm(Layout::kColMajor, a_transpose, Transpose::kNo,
+                     m - i - block_size, n, block_size, ConstantNegOne<T>(),
+                     a_buffer, this_a_offset + a_offset, a_ld,
+                     x_buffer, x_offset + i, x_ld, gemm_alpha,
+                     b_buffer, b_offset + i + block_size, b_ld);
+        gemm2_event.WaitForCompletion();
       }
     }
 
@@ -150,18 +157,25 @@ void Xtrsm<T>::TrsmColMajor(const Side side, const Triangle triangle,
       for (auto i = i_start; i >= 0; i -= static_cast<int>(block_size)) {
         const auto current_block_size = (i == i_start) ? special_block_size : block_size;
         const auto gemm_alpha = (i == i_start) ? alpha : ConstantOne<T>();
-        DoGemm(Layout::kColMajor, a_transpose, Transpose::kNo,
-               current_block_size, n, current_block_size, gemm_alpha,
-               a_inv_buffer, i * block_size, block_size,
-               b_buffer, b_offset + i, b_ld, ConstantZero<T>(),
-               x_buffer, x_offset + i, x_ld);
+        auto gemm1_event = Event();
+        auto gemm1 = Xgemm<T>(queue_, gemm1_event.pointer());
+        gemm1.DoGemm(Layout::kColMajor, a_transpose, Transpose::kNo,
+                     current_block_size, n, current_block_size, gemm_alpha,
+                     a_inv_buffer, i * block_size, block_size,
+                     b_buffer, b_offset + i, b_ld, ConstantZero<T>(),
+                     x_buffer, x_offset + i, x_ld);
+        gemm1_event.WaitForCompletion();
         if (i - static_cast<int>(block_size) < 0) { break; }
+
         const auto this_a_offset = (a_transpose == Transpose::kNo) ? i * a_ld : i;
-        DoGemm(Layout::kColMajor, a_transpose, Transpose::kNo,
-               i, n, current_block_size, ConstantNegOne<T>(),
-               a_buffer, this_a_offset, a_ld,
-               x_buffer, x_offset + i, x_ld, gemm_alpha,
-               b_buffer, b_offset, b_ld);
+        auto gemm2_event = Event();
+        auto gemm2 = Xgemm<T>(queue_, gemm2_event.pointer());
+        gemm2.DoGemm(Layout::kColMajor, a_transpose, Transpose::kNo,
+                     i, n, current_block_size, ConstantNegOne<T>(),
+                     a_buffer, this_a_offset + a_offset, a_ld,
+                     x_buffer, x_offset + i, x_ld, gemm_alpha,
+                     b_buffer, b_offset, b_ld);
+        gemm2_event.WaitForCompletion();
       }
     }
   }
@@ -176,18 +190,25 @@ void Xtrsm<T>::TrsmColMajor(const Side side, const Triangle triangle,
       for (auto i = i_start; i >= 0; i -= static_cast<int>(block_size)) {
         const auto current_block_size = (i == i_start) ? special_block_size : block_size;
         const auto gemm_alpha = (i == i_start) ? alpha : ConstantOne<T>();
-        DoGemm(Layout::kColMajor, Transpose::kNo, a_transpose,
-               m, current_block_size, current_block_size, gemm_alpha,
-               b_buffer, b_offset + i * b_ld, b_ld,
-               a_inv_buffer, i * block_size, block_size, ConstantZero<T>(),
-               x_buffer, x_offset + i * x_ld, x_ld);
+        auto gemm1_event = Event();
+        auto gemm1 = Xgemm<T>(queue_, gemm1_event.pointer());
+        gemm1.DoGemm(Layout::kColMajor, Transpose::kNo, a_transpose,
+                     m, current_block_size, current_block_size, gemm_alpha,
+                     b_buffer, b_offset + i * b_ld, b_ld,
+                     a_inv_buffer, i * block_size, block_size, ConstantZero<T>(),
+                     x_buffer, x_offset + i * x_ld, x_ld);
+        gemm1_event.WaitForCompletion();
         if (i - static_cast<int>(block_size) < 0) { break; }
+
         const auto this_a_offset = (a_transpose == Transpose::kNo) ? i : i * a_ld;
-        DoGemm(Layout::kColMajor, Transpose::kNo, a_transpose,
-               m, i, current_block_size, ConstantNegOne<T>(),
-               x_buffer, x_offset + i * x_ld, x_ld,
-               a_buffer, this_a_offset, a_ld, gemm_alpha,
-               b_buffer, b_offset, b_ld);
+        auto gemm2_event = Event();
+        auto gemm2 = Xgemm<T>(queue_, gemm2_event.pointer());
+        gemm2.DoGemm(Layout::kColMajor, Transpose::kNo, a_transpose,
+                     m, i, current_block_size, ConstantNegOne<T>(),
+                     x_buffer, x_offset + i * x_ld, x_ld,
+                     a_buffer, this_a_offset + a_offset, a_ld, gemm_alpha,
+                     b_buffer, b_offset, b_ld);
+        gemm2_event.WaitForCompletion();
       }
     }
 
@@ -196,18 +217,25 @@ void Xtrsm<T>::TrsmColMajor(const Side side, const Triangle triangle,
       for (auto i = size_t{0}; i < n; i += block_size) {
         const auto gemm_alpha = (i == 0) ? alpha : ConstantOne<T>();
         const auto current_block_size = std::min(n - i, block_size);
-        DoGemm(Layout::kColMajor, Transpose::kNo, a_transpose,
-               m, current_block_size, current_block_size, gemm_alpha,
-               b_buffer, b_offset + i * b_ld, b_ld,
-               a_inv_buffer, i * block_size, block_size, ConstantZero<T>(),
-               x_buffer, x_offset + i * x_ld, x_ld);
+        auto gemm1_event = Event();
+        auto gemm1 = Xgemm<T>(queue_, gemm1_event.pointer());
+        gemm1.DoGemm(Layout::kColMajor, Transpose::kNo, a_transpose,
+                     m, current_block_size, current_block_size, gemm_alpha,
+                     b_buffer, b_offset + i * b_ld, b_ld,
+                     a_inv_buffer, i * block_size, block_size, ConstantZero<T>(),
+                     x_buffer, x_offset + i * x_ld, x_ld);
+        gemm1_event.WaitForCompletion();
         if (i + block_size >= n) { break; }
+
         const auto this_a_offset = (a_transpose == Transpose::kNo) ? i + (block_size + i) * a_ld : (i + block_size) + i * a_ld;
-        DoGemm(Layout::kColMajor, Transpose::kNo, a_transpose,
-               m, n - i - block_size, block_size, ConstantNegOne<T>(),
-               x_buffer, x_offset + i * x_ld, x_ld,
-               a_buffer, this_a_offset, a_ld, gemm_alpha,
-               b_buffer, b_offset + (i + block_size) * b_ld, b_ld);
+        auto gemm2_event = Event();
+        auto gemm2 = Xgemm<T>(queue_, gemm2_event.pointer());
+        gemm2.DoGemm(Layout::kColMajor, Transpose::kNo, a_transpose,
+                     m, n - i - block_size, block_size, ConstantNegOne<T>(),
+                     x_buffer, x_offset + i * x_ld, x_ld,
+                     a_buffer, this_a_offset + a_offset, a_ld, gemm_alpha,
+                     b_buffer, b_offset + (i + block_size) * b_ld, b_ld);
+        gemm2_event.WaitForCompletion();
       }
     }
   }