From 4507ba4997cd546418eae0972c018073ac7b36aa Mon Sep 17 00:00:00 2001
From: CNugteren <web@cedricnugteren.nl>
Date: Fri, 18 Sep 2015 15:25:20 +0200
Subject: Added first version of banded matrix-vector multiplication

---
 src/clblast.cc                                  |  30 +-
 src/kernels/level2/xgemv.opencl                 | 395 ++++++++++++++++++++++++
 src/kernels/matrix_transforms/gbgemt.opencl     |  60 ++++
 src/kernels/matrix_transforms/transforms.opencl |  40 +++
 src/kernels/xgemv.opencl                        | 395 ------------------------
 src/routines/level2/xgbmv.cc                    | 117 +++++++
 src/routines/level2/xgemv.cc                    |   6 +-
 src/tuning/xgemv.cc                             |   2 +-
 8 files changed, 638 insertions(+), 407 deletions(-)
 create mode 100644 src/kernels/level2/xgemv.opencl
 create mode 100644 src/kernels/matrix_transforms/gbgemt.opencl
 create mode 100644 src/kernels/matrix_transforms/transforms.opencl
 delete mode 100644 src/kernels/xgemv.opencl
 create mode 100644 src/routines/level2/xgbmv.cc

(limited to 'src')

diff --git a/src/clblast.cc b/src/clblast.cc
index a0dd8c70..ad5e354d 100644
--- a/src/clblast.cc
+++ b/src/clblast.cc
@@ -28,6 +28,7 @@
 
 // BLAS level-2 includes
 #include "internal/routines/level2/xgemv.h"
+#include "internal/routines/level2/xgbmv.h"
 #include "internal/routines/level2/xhemv.h"
 #include "internal/routines/level2/xsymv.h"
 
@@ -327,15 +328,26 @@ template StatusCode Gemv<double2>(const Layout, const Transpose,
 
 // General banded matrix-vector multiplication: SGBMV/DGBMV/CGBMV/ZGBMV
 template <typename T>
-StatusCode Gbmv(const Layout, const Transpose,
-                const size_t, const size_t, const size_t, const size_t,
-                const T,
-                const cl_mem, const size_t, const size_t,
-                const cl_mem, const size_t, const size_t,
-                const T,
-                cl_mem, const size_t, const size_t,
-                cl_command_queue*, cl_event*) {
-  return StatusCode::kNotImplemented;
+StatusCode Gbmv(const Layout layout, const Transpose a_transpose,
+                const size_t m, const size_t n, const size_t kl, const size_t ku,
+                const T alpha,
+                const cl_mem a_buffer, const size_t a_offset, const size_t a_ld,
+                const cl_mem x_buffer, const size_t x_offset, const size_t x_inc,
+                const T beta,
+                cl_mem y_buffer, const size_t y_offset, const size_t y_inc,
+                cl_command_queue* queue, cl_event* event) {
+  auto queue_cpp = Queue(*queue);
+  auto event_cpp = Event(*event);
+  auto routine = Xgbmv<T>(queue_cpp, event_cpp);
+  auto status = routine.SetUp();
+  if (status != StatusCode::kSuccess) { return status; }
+  return routine.DoGbmv(layout, a_transpose,
+                        m, n, kl, ku,
+                        alpha,
+                        Buffer<T>(a_buffer), a_offset, a_ld,
+                        Buffer<T>(x_buffer), x_offset, x_inc,
+                        beta,
+                        Buffer<T>(y_buffer), y_offset, y_inc);
 }
 template StatusCode Gbmv<float>(const Layout, const Transpose,
                                 const size_t, const size_t, const size_t, const size_t,
diff --git a/src/kernels/level2/xgemv.opencl b/src/kernels/level2/xgemv.opencl
new file mode 100644
index 00000000..1e12dd78
--- /dev/null
+++ b/src/kernels/level2/xgemv.opencl
@@ -0,0 +1,395 @@
+
+// =================================================================================================
+// This file is part of the CLBlast project. The project is licensed under Apache Version 2.0. This
+// project loosely follows the Google C++ styleguide and uses a tab-size of two spaces and a max-
+// width of 100 characters per line.
+//
+// Author(s):
+//   Cedric Nugteren <www.cedricnugteren.nl>
+//
+// This file contains the Xgemv kernel for matrix-vector multiplication.
+//
+// =================================================================================================
+
+// Enables loading of this file using the C++ pre-processor's #include (C++11 standard raw string
+// literal). Comment-out this line for syntax-highlighting when developing.
+R"(
+
+// =================================================================================================
+
+// Parameters set by the tuner or by the database. Here they are given a basic default value in case
+// this kernel file is used outside of the CLBlast library.
+
+// 1: For the full version of the kernel
+#ifndef WGS1
+  #define WGS1 64     // The local work-group size
+#endif
+#ifndef WPT1
+  #define WPT1 1      // The amount of work-per-thread
+#endif
+
+// 2: For the fast version
+#ifndef WGS2
+  #define WGS2 64     // The local work-group size
+#endif
+#ifndef WPT2
+  #define WPT2 1      // The amount of work-per-thread
+#endif
+#ifndef VW2
+  #define VW2 1       // Vector width of matrix A loads
+#endif
+
+// 3: For the fast rotated version
+#ifndef WGS3
+  #define WGS3 64     // The local work-group size
+#endif
+#ifndef WPT3
+  #define WPT3 1      // The amount of work-per-thread
+#endif
+#ifndef VW3
+  #define VW3 1       // Vector width of matrix A loads
+#endif
+
+// =================================================================================================
+
+// Data-widths for the 'fast' kernel
+#if VW2 == 1
+  typedef real realVF;
+#elif VW2 == 2
+  typedef real2 realVF;
+#elif VW2 == 4
+  typedef real4 realVF;
+#elif VW2 == 8
+  typedef real8 realVF;
+#elif VW2 == 16
+  typedef real16 realVF;
+#endif
+
+// Data-widths for the 'fast' kernel with rotated matrix
+#if VW3 == 1
+  typedef real realVFR;
+#elif VW3 == 2
+  typedef real2 realVFR;
+#elif VW3 == 4
+  typedef real4 realVFR;
+#elif VW3 == 8
+  typedef real8 realVFR;
+#elif VW3 == 16
+  typedef real16 realVFR;
+#endif
+
+// =================================================================================================
+// Defines how to load the input matrix in the regular case
+
+// Loads a scalar input value
+inline real LoadMatrixA(const __global real* restrict agm, const int x, const int y,
+                        const int a_ld, const int a_offset) {
+  return agm[x + a_ld*y + a_offset];
+}
+// Loads a vector input value (1/2)
+inline realVF LoadMatrixAVF(const __global realVF* restrict agm, const int x, const int y,
+                            const int a_ld) {
+  return agm[x + a_ld*y];
+}
+// Loads a vector input value (2/2): as before, but different data-type
+inline realVFR LoadMatrixAVFR(const __global realVFR* restrict agm, const int x, const int y,
+                              const int a_ld) {
+  return agm[x + a_ld*y];
+}
+
+// =================================================================================================
+
+// Full version of the kernel
+__attribute__((reqd_work_group_size(WGS1, 1, 1)))
+__kernel void Xgemv(const int m, const int n, const real alpha, const real beta,
+                    const int a_rotated,
+                    const __global real* restrict agm, const int a_offset, const int a_ld,
+                    const __global real* restrict xgm, const int x_offset, const int x_inc,
+                    __global real* ygm, const int y_offset, const int y_inc,
+                    const int do_conjugate) {
+
+  // Local memory for the vector X
+  __local real xlm[WGS1];
+
+  // Initializes the accumulation register
+  real acc[WPT1];
+  #pragma unroll
+  for (int w=0; w<WPT1; ++w) {
+    SetToZero(acc[w]);
+  }
+
+  // Divides the work in a main and tail section
+  const int n_tail = n % WGS1;
+  const int n_floor = n - n_tail;
+
+  // Loops over work-group sized portions of the work
+  for (int kwg=0; kwg<n_floor; kwg+=WGS1) {
+
+    // Loads the vector X into local memory
+    const int lid = get_local_id(0);
+    xlm[lid] = xgm[(kwg + lid)*x_inc + x_offset];
+
+    // Synchronizes all threads in a workgroup
+    barrier(CLK_LOCAL_MEM_FENCE);
+
+    // Loops over the work per thread, and checks whether in bounds
+    #pragma unroll
+    for (int w=0; w<WPT1; ++w) {
+      const int gid = w*get_global_size(0) + get_global_id(0);
+      if (gid < m) {
+
+        // The multiply-add function for the main part (divisable by WGS1)
+        if (a_rotated == 0) { // Not rotated
+          #pragma unroll
+          for (int kl=0; kl<WGS1; ++kl) {
+            const int k = kwg + kl;
+            real value = LoadMatrixA(agm, gid, k, a_ld, a_offset);
+            if (do_conjugate == 1) { COMPLEX_CONJUGATE(value); }
+            MultiplyAdd(acc[w], xlm[kl], value);
+          }
+        }
+        else { // Transposed
+          #pragma unroll
+          for (int kl=0; kl<WGS1; ++kl) {
+            const int k = kwg + kl;
+            real value = LoadMatrixA(agm, k, gid, a_ld, a_offset);
+            if (do_conjugate == 1) { COMPLEX_CONJUGATE(value); }
+            MultiplyAdd(acc[w], xlm[kl], value);
+          }
+        }
+      }
+    }
+
+    // Synchronizes all threads in a workgroup
+    barrier(CLK_LOCAL_MEM_FENCE);
+  }
+
+  // Loops over the work per thread, and checks whether in bounds
+  #pragma unroll
+  for (int w=0; w<WPT1; ++w) {
+    const int gid = w*get_global_size(0) + get_global_id(0);
+    if (gid < m) {
+
+      // The multiply-add function for the remainder part (not divisable by WGS1)
+      if (a_rotated == 0) { // Not rotated
+        #pragma unroll
+        for (int k=n_floor; k<n; ++k) {
+          real value = LoadMatrixA(agm, gid, k, a_ld, a_offset);
+          if (do_conjugate == 1) { COMPLEX_CONJUGATE(value); }
+          MultiplyAdd(acc[w], xgm[k*x_inc + x_offset], value);
+        }
+      }
+      else { // Transposed
+        #pragma unroll
+        for (int k=n_floor; k<n; ++k) {
+          real value = LoadMatrixA(agm, k, gid, a_ld, a_offset);
+          if (do_conjugate == 1) { COMPLEX_CONJUGATE(value); }
+          MultiplyAdd(acc[w], xgm[k*x_inc + x_offset], value);
+        }
+      }
+
+      // Stores the final result
+      real yval = ygm[gid*y_inc + y_offset];
+      AXPBY(ygm[gid*y_inc + y_offset], alpha, acc[w], beta, yval);
+    }
+  }
+}
+
+// =================================================================================================
+
+// Faster version of the kernel, assuming that:
+// --> 'm' and 'n' are multiples of WGS2
+// --> 'a_offset' is 0
+// --> 'a_ld' is a multiple of VW2
+// --> 'a_rotated' is 0
+// --> 'do_conjugate' is 0
+__attribute__((reqd_work_group_size(WGS2, 1, 1)))
+__kernel void XgemvFast(const int m, const int n, const real alpha, const real beta,
+                        const int a_rotated,
+                        const __global realVF* restrict agm, const int a_offset, const int a_ld,
+                        const __global real* restrict xgm, const int x_offset, const int x_inc,
+                        __global real* ygm, const int y_offset, const int y_inc,
+                        const int do_conjugate) {
+  // Local memory for the vector X
+  __local real xlm[WGS2];
+
+  // Initializes the accumulation register
+  real acc[WPT2];
+  #pragma unroll
+  for (int w=0; w<WPT2; ++w) {
+    SetToZero(acc[w]);
+  }
+
+  // Loops over work-group sized portions of the work
+  for (int kwg=0; kwg<n; kwg+=WGS2) {
+
+    // Loads the vector X into local memory
+    const int lid = get_local_id(0);
+    xlm[lid] = xgm[(kwg + lid)*x_inc + x_offset];
+
+    // Synchronizes all threads in a workgroup
+    barrier(CLK_LOCAL_MEM_FENCE);
+
+    // The multiply-add function (not rotated)
+    #pragma unroll
+    for (int kl=0; kl<WGS2; ++kl) {
+      const int k = kwg + kl;
+      #pragma unroll
+      for (int w=0; w<WPT2/VW2; ++w) {
+        const int gid = (WPT2/VW2)*get_global_id(0) + w;
+        realVF avec = LoadMatrixAVF(agm, gid, k, a_ld/VW2);
+        #if VW2 == 1
+          MultiplyAdd(acc[VW2*w+0], xlm[kl], avec);
+        #elif VW2 == 2
+          MultiplyAdd(acc[VW2*w+0], xlm[kl], avec.x);
+          MultiplyAdd(acc[VW2*w+1], xlm[kl], avec.y);
+        #elif VW2 == 4
+          MultiplyAdd(acc[VW2*w+0], xlm[kl], avec.x);
+          MultiplyAdd(acc[VW2*w+1], xlm[kl], avec.y);
+          MultiplyAdd(acc[VW2*w+2], xlm[kl], avec.z);
+          MultiplyAdd(acc[VW2*w+3], xlm[kl], avec.w);
+        #elif VW2 == 8
+          MultiplyAdd(acc[VW2*w+0], xlm[kl], avec.s0);
+          MultiplyAdd(acc[VW2*w+1], xlm[kl], avec.s1);
+          MultiplyAdd(acc[VW2*w+2], xlm[kl], avec.s2);
+          MultiplyAdd(acc[VW2*w+3], xlm[kl], avec.s3);
+          MultiplyAdd(acc[VW2*w+4], xlm[kl], avec.s4);
+          MultiplyAdd(acc[VW2*w+5], xlm[kl], avec.s5);
+          MultiplyAdd(acc[VW2*w+6], xlm[kl], avec.s6);
+          MultiplyAdd(acc[VW2*w+7], xlm[kl], avec.s7);
+        #elif VW2 == 16
+          MultiplyAdd(acc[VW2*w+0], xlm[kl], avec.s0);
+          MultiplyAdd(acc[VW2*w+1], xlm[kl], avec.s1);
+          MultiplyAdd(acc[VW2*w+2], xlm[kl], avec.s2);
+          MultiplyAdd(acc[VW2*w+3], xlm[kl], avec.s3);
+          MultiplyAdd(acc[VW2*w+4], xlm[kl], avec.s4);
+          MultiplyAdd(acc[VW2*w+5], xlm[kl], avec.s5);
+          MultiplyAdd(acc[VW2*w+6], xlm[kl], avec.s6);
+          MultiplyAdd(acc[VW2*w+7], xlm[kl], avec.s7);
+          MultiplyAdd(acc[VW2*w+8], xlm[kl], avec.s8);
+          MultiplyAdd(acc[VW2*w+9], xlm[kl], avec.s9);
+          MultiplyAdd(acc[VW2*w+10], xlm[kl], avec.sA);
+          MultiplyAdd(acc[VW2*w+11], xlm[kl], avec.sB);
+          MultiplyAdd(acc[VW2*w+12], xlm[kl], avec.sC);
+          MultiplyAdd(acc[VW2*w+13], xlm[kl], avec.sD);
+          MultiplyAdd(acc[VW2*w+14], xlm[kl], avec.sE);
+          MultiplyAdd(acc[VW2*w+15], xlm[kl], avec.sF);
+        #endif
+      }
+    }
+
+    // Synchronizes all threads in a workgroup
+    barrier(CLK_LOCAL_MEM_FENCE);
+  }
+
+  // Stores the final result
+  #pragma unroll
+  for (int w=0; w<WPT2; ++w) {
+    const int gid = WPT2*get_global_id(0) + w;
+    real yval = ygm[gid*y_inc + y_offset];
+    AXPBY(ygm[gid*y_inc + y_offset], alpha, acc[w], beta, yval);
+  }
+}
+
+// =================================================================================================
+
+// Faster version of the kernel, assuming that:
+// --> 'm' and 'n' are multiples of WGS3
+// --> 'a_offset' is 0
+// --> 'a_ld' is a multiple of VW3
+// --> 'a_rotated' is 1
+// --> 'do_conjugate' is 0
+__attribute__((reqd_work_group_size(WGS3, 1, 1)))
+__kernel void XgemvFastRot(const int m, const int n, const real alpha, const real beta,
+                           const int a_rotated,
+                           const __global realVFR* restrict agm, const int a_offset, const int a_ld,
+                           const __global real* restrict xgm, const int x_offset, const int x_inc,
+                           __global real* ygm, const int y_offset, const int y_inc,
+                           const int do_conjugate) {
+  // Local memory for the vector X
+  __local real xlm[WGS3];
+
+  // Initializes the accumulation register
+  real acc[WPT3];
+  #pragma unroll
+  for (int w=0; w<WPT3; ++w) {
+    SetToZero(acc[w]);
+  }
+
+  // Loops over work-group sized portions of the work
+  for (int kwg=0; kwg<n; kwg+=WGS3) {
+
+    // Loads the vector X into local memory
+    const int lid = get_local_id(0);
+    xlm[lid] = xgm[(kwg + lid)*x_inc + x_offset];
+
+    // Synchronizes all threads in a workgroup
+    barrier(CLK_LOCAL_MEM_FENCE);
+
+    // The multiply-add function (rotated)
+    #pragma unroll
+    for (int kl=0; kl<WGS3/VW3; ++kl) {
+      const int k = (kwg/VW3) + kl;
+      #pragma unroll
+      for (int w=0; w<WPT3; ++w) {
+        const int gid = WPT3*get_global_id(0) + w;
+        realVFR avec = LoadMatrixAVFR(agm, k, gid, a_ld/VW3);
+        #if VW3 == 1
+          MultiplyAdd(acc[w], xlm[VW3*kl+0], avec);
+        #elif VW3 == 2
+          MultiplyAdd(acc[w], xlm[VW3*kl+0], avec.x);
+          MultiplyAdd(acc[w], xlm[VW3*kl+1], avec.y);
+        #elif VW3 == 4
+          MultiplyAdd(acc[w], xlm[VW3*kl+0], avec.x);
+          MultiplyAdd(acc[w], xlm[VW3*kl+1], avec.y);
+          MultiplyAdd(acc[w], xlm[VW3*kl+2], avec.z);
+          MultiplyAdd(acc[w], xlm[VW3*kl+3], avec.w);
+        #elif VW3 == 8
+          MultiplyAdd(acc[w], xlm[VW3*kl+0], avec.s0);
+          MultiplyAdd(acc[w], xlm[VW3*kl+1], avec.s1);
+          MultiplyAdd(acc[w], xlm[VW3*kl+2], avec.s2);
+          MultiplyAdd(acc[w], xlm[VW3*kl+3], avec.s3);
+          MultiplyAdd(acc[w], xlm[VW3*kl+4], avec.s4);
+          MultiplyAdd(acc[w], xlm[VW3*kl+5], avec.s5);
+          MultiplyAdd(acc[w], xlm[VW3*kl+6], avec.s6);
+          MultiplyAdd(acc[w], xlm[VW3*kl+7], avec.s7);
+        #elif VW3 == 16
+          MultiplyAdd(acc[w], xlm[VW3*kl+0], avec.s0);
+          MultiplyAdd(acc[w], xlm[VW3*kl+1], avec.s1);
+          MultiplyAdd(acc[w], xlm[VW3*kl+2], avec.s2);
+          MultiplyAdd(acc[w], xlm[VW3*kl+3], avec.s3);
+          MultiplyAdd(acc[w], xlm[VW3*kl+4], avec.s4);
+          MultiplyAdd(acc[w], xlm[VW3*kl+5], avec.s5);
+          MultiplyAdd(acc[w], xlm[VW3*kl+6], avec.s6);
+          MultiplyAdd(acc[w], xlm[VW3*kl+7], avec.s7);
+          MultiplyAdd(acc[w], xlm[VW3*kl+8], avec.s8);
+          MultiplyAdd(acc[w], xlm[VW3*kl+9], avec.s9);
+          MultiplyAdd(acc[w], xlm[VW3*kl+10], avec.sA);
+          MultiplyAdd(acc[w], xlm[VW3*kl+11], avec.sB);
+          MultiplyAdd(acc[w], xlm[VW3*kl+12], avec.sC);
+          MultiplyAdd(acc[w], xlm[VW3*kl+13], avec.sD);
+          MultiplyAdd(acc[w], xlm[VW3*kl+14], avec.sE);
+          MultiplyAdd(acc[w], xlm[VW3*kl+15], avec.sF);
+        #endif
+      }
+    }
+
+    // Synchronizes all threads in a workgroup
+    barrier(CLK_LOCAL_MEM_FENCE);
+  }
+
+  // Stores the final result
+  #pragma unroll
+  for (int w=0; w<WPT3; ++w) {
+    const int gid = WPT3*get_global_id(0) + w;
+    real yval = ygm[gid*y_inc + y_offset];
+    AXPBY(ygm[gid*y_inc + y_offset], alpha, acc[w], beta, yval);
+  }
+}
+
+// =================================================================================================
+
+// End of the C++11 raw string literal
+)"
+
+// =================================================================================================
diff --git a/src/kernels/matrix_transforms/gbgemt.opencl b/src/kernels/matrix_transforms/gbgemt.opencl
new file mode 100644
index 00000000..e46e3a59
--- /dev/null
+++ b/src/kernels/matrix_transforms/gbgemt.opencl
@@ -0,0 +1,60 @@
+
+// =================================================================================================
+// This file is part of the CLBlast project. The project is licensed under Apache Version 2.0. This
+// project loosely follows the Google C++ styleguide and uses a tab-size of two spaces and a max-
+// width of 100 characters per line.
+//
+// Author(s):
+//   Cedric Nugteren <www.cedricnugteren.nl>
+//
+// This file contains the general banded (gb) to general (ge) matrix transforms.
+//
+// This kernel uses the matrix-transforms common tuning parameters.
+//
+// =================================================================================================
+
+// Enables loading of this file using the C++ pre-processor's #include (C++11 standard raw string
+// literal). Comment-out this line for syntax-highlighting when developing.
+R"(
+
+// =================================================================================================
+#if defined(ROUTINE_GBMV)
+
+// Kernel to transform a general banded matrix into a general matrix
+__attribute__((reqd_work_group_size(PAD_DIMX, PAD_DIMY, 1)))
+__kernel void GeneralBandedToGeneral(const int src_one, const int src_two,
+                                     const int src_ld, const int src_offset,
+                                     __global const real* restrict src,
+                                     const int dest_one, const int dest_two,
+                                     const int dest_ld, const int dest_offset,
+                                     __global real* dest,
+                                     const int layout,
+                                     const int kl, const int ku) {
+
+  // Loops over the work per thread in both dimensions
+  #pragma unroll
+  for (int w_one=0; w_one<PAD_WPTX; ++w_one) {
+    const int id_one = (get_group_id(0)*PAD_WPTX + w_one) * PAD_DIMX + get_local_id(0);
+    #pragma unroll
+    for (int w_two=0; w_two<PAD_WPTY; ++w_two) {
+      const int id_two = (get_group_id(1)*PAD_WPTY + w_two) * PAD_DIMY + get_local_id(1);
+      if (id_two < dest_two && id_one < dest_one) {
+        real result;
+        SetToZero(result);
+        const int k = ku - id_two + id_one;
+        if ((id_one >= id_two - ku) && (id_one < id_two + kl + 1)) {
+          result = src[id_two*src_ld + k + src_offset];
+        }
+        dest[id_two*dest_ld + id_one + dest_offset] = result;
+      }
+    }
+  }
+}
+
+#endif
+// =================================================================================================
+
+// End of the C++11 raw string literal
+)"
+
+// =================================================================================================
diff --git a/src/kernels/matrix_transforms/transforms.opencl b/src/kernels/matrix_transforms/transforms.opencl
new file mode 100644
index 00000000..01889a13
--- /dev/null
+++ b/src/kernels/matrix_transforms/transforms.opencl
@@ -0,0 +1,40 @@
+
+// =================================================================================================
+// This file is part of the CLBlast project. The project is licensed under Apache Version 2.0. This
+// project loosely follows the Google C++ styleguide and uses a tab-size of two spaces and a max-
+// width of 100 characters per line.
+//
+// Author(s):
+//   Cedric Nugteren <www.cedricnugteren.nl>
+//
+// This file contains the common functions and parameters specific for matrix-transform kernels.
+//
+// =================================================================================================
+
+// Enables loading of this file using the C++ pre-processor's #include (C++11 standard raw string
+// literal). Comment-out this line for syntax-highlighting when developing.
+R"(
+
+// =================================================================================================
+
+// Parameters set by the tuner or by the database. Here they are given a basic default value in case
+// this kernel file is used outside of the CLBlast library.
+#ifndef PAD_DIMX
+  #define PAD_DIMX 8      // Local workgroup size in the first dimension (x)
+#endif
+#ifndef PAD_DIMY
+  #define PAD_DIMY 8      // Local workgroup size in the second dimension (y)
+#endif
+#ifndef PAD_WPTX
+  #define PAD_WPTX 1      // Work per thread in the first dimension (x)
+#endif
+#ifndef PAD_WPTY
+  #define PAD_WPTY 1      // Work per thread in the second dimension (y)
+#endif
+
+// =================================================================================================
+
+// End of the C++11 raw string literal
+)"
+
+// =================================================================================================
diff --git a/src/kernels/xgemv.opencl b/src/kernels/xgemv.opencl
deleted file mode 100644
index 1e12dd78..00000000
--- a/src/kernels/xgemv.opencl
+++ /dev/null
@@ -1,395 +0,0 @@
-
-// =================================================================================================
-// This file is part of the CLBlast project. The project is licensed under Apache Version 2.0. This
-// project loosely follows the Google C++ styleguide and uses a tab-size of two spaces and a max-
-// width of 100 characters per line.
-//
-// Author(s):
-//   Cedric Nugteren <www.cedricnugteren.nl>
-//
-// This file contains the Xgemv kernel for matrix-vector multiplication.
-//
-// =================================================================================================
-
-// Enables loading of this file using the C++ pre-processor's #include (C++11 standard raw string
-// literal). Comment-out this line for syntax-highlighting when developing.
-R"(
-
-// =================================================================================================
-
-// Parameters set by the tuner or by the database. Here they are given a basic default value in case
-// this kernel file is used outside of the CLBlast library.
-
-// 1: For the full version of the kernel
-#ifndef WGS1
-  #define WGS1 64     // The local work-group size
-#endif
-#ifndef WPT1
-  #define WPT1 1      // The amount of work-per-thread
-#endif
-
-// 2: For the fast version
-#ifndef WGS2
-  #define WGS2 64     // The local work-group size
-#endif
-#ifndef WPT2
-  #define WPT2 1      // The amount of work-per-thread
-#endif
-#ifndef VW2
-  #define VW2 1       // Vector width of matrix A loads
-#endif
-
-// 3: For the fast rotated version
-#ifndef WGS3
-  #define WGS3 64     // The local work-group size
-#endif
-#ifndef WPT3
-  #define WPT3 1      // The amount of work-per-thread
-#endif
-#ifndef VW3
-  #define VW3 1       // Vector width of matrix A loads
-#endif
-
-// =================================================================================================
-
-// Data-widths for the 'fast' kernel
-#if VW2 == 1
-  typedef real realVF;
-#elif VW2 == 2
-  typedef real2 realVF;
-#elif VW2 == 4
-  typedef real4 realVF;
-#elif VW2 == 8
-  typedef real8 realVF;
-#elif VW2 == 16
-  typedef real16 realVF;
-#endif
-
-// Data-widths for the 'fast' kernel with rotated matrix
-#if VW3 == 1
-  typedef real realVFR;
-#elif VW3 == 2
-  typedef real2 realVFR;
-#elif VW3 == 4
-  typedef real4 realVFR;
-#elif VW3 == 8
-  typedef real8 realVFR;
-#elif VW3 == 16
-  typedef real16 realVFR;
-#endif
-
-// =================================================================================================
-// Defines how to load the input matrix in the regular case
-
-// Loads a scalar input value
-inline real LoadMatrixA(const __global real* restrict agm, const int x, const int y,
-                        const int a_ld, const int a_offset) {
-  return agm[x + a_ld*y + a_offset];
-}
-// Loads a vector input value (1/2)
-inline realVF LoadMatrixAVF(const __global realVF* restrict agm, const int x, const int y,
-                            const int a_ld) {
-  return agm[x + a_ld*y];
-}
-// Loads a vector input value (2/2): as before, but different data-type
-inline realVFR LoadMatrixAVFR(const __global realVFR* restrict agm, const int x, const int y,
-                              const int a_ld) {
-  return agm[x + a_ld*y];
-}
-
-// =================================================================================================
-
-// Full version of the kernel
-__attribute__((reqd_work_group_size(WGS1, 1, 1)))
-__kernel void Xgemv(const int m, const int n, const real alpha, const real beta,
-                    const int a_rotated,
-                    const __global real* restrict agm, const int a_offset, const int a_ld,
-                    const __global real* restrict xgm, const int x_offset, const int x_inc,
-                    __global real* ygm, const int y_offset, const int y_inc,
-                    const int do_conjugate) {
-
-  // Local memory for the vector X
-  __local real xlm[WGS1];
-
-  // Initializes the accumulation register
-  real acc[WPT1];
-  #pragma unroll
-  for (int w=0; w<WPT1; ++w) {
-    SetToZero(acc[w]);
-  }
-
-  // Divides the work in a main and tail section
-  const int n_tail = n % WGS1;
-  const int n_floor = n - n_tail;
-
-  // Loops over work-group sized portions of the work
-  for (int kwg=0; kwg<n_floor; kwg+=WGS1) {
-
-    // Loads the vector X into local memory
-    const int lid = get_local_id(0);
-    xlm[lid] = xgm[(kwg + lid)*x_inc + x_offset];
-
-    // Synchronizes all threads in a workgroup
-    barrier(CLK_LOCAL_MEM_FENCE);
-
-    // Loops over the work per thread, and checks whether in bounds
-    #pragma unroll
-    for (int w=0; w<WPT1; ++w) {
-      const int gid = w*get_global_size(0) + get_global_id(0);
-      if (gid < m) {
-
-        // The multiply-add function for the main part (divisable by WGS1)
-        if (a_rotated == 0) { // Not rotated
-          #pragma unroll
-          for (int kl=0; kl<WGS1; ++kl) {
-            const int k = kwg + kl;
-            real value = LoadMatrixA(agm, gid, k, a_ld, a_offset);
-            if (do_conjugate == 1) { COMPLEX_CONJUGATE(value); }
-            MultiplyAdd(acc[w], xlm[kl], value);
-          }
-        }
-        else { // Transposed
-          #pragma unroll
-          for (int kl=0; kl<WGS1; ++kl) {
-            const int k = kwg + kl;
-            real value = LoadMatrixA(agm, k, gid, a_ld, a_offset);
-            if (do_conjugate == 1) { COMPLEX_CONJUGATE(value); }
-            MultiplyAdd(acc[w], xlm[kl], value);
-          }
-        }
-      }
-    }
-
-    // Synchronizes all threads in a workgroup
-    barrier(CLK_LOCAL_MEM_FENCE);
-  }
-
-  // Loops over the work per thread, and checks whether in bounds
-  #pragma unroll
-  for (int w=0; w<WPT1; ++w) {
-    const int gid = w*get_global_size(0) + get_global_id(0);
-    if (gid < m) {
-
-      // The multiply-add function for the remainder part (not divisable by WGS1)
-      if (a_rotated == 0) { // Not rotated
-        #pragma unroll
-        for (int k=n_floor; k<n; ++k) {
-          real value = LoadMatrixA(agm, gid, k, a_ld, a_offset);
-          if (do_conjugate == 1) { COMPLEX_CONJUGATE(value); }
-          MultiplyAdd(acc[w], xgm[k*x_inc + x_offset], value);
-        }
-      }
-      else { // Transposed
-        #pragma unroll
-        for (int k=n_floor; k<n; ++k) {
-          real value = LoadMatrixA(agm, k, gid, a_ld, a_offset);
-          if (do_conjugate == 1) { COMPLEX_CONJUGATE(value); }
-          MultiplyAdd(acc[w], xgm[k*x_inc + x_offset], value);
-        }
-      }
-
-      // Stores the final result
-      real yval = ygm[gid*y_inc + y_offset];
-      AXPBY(ygm[gid*y_inc + y_offset], alpha, acc[w], beta, yval);
-    }
-  }
-}
-
-// =================================================================================================
-
-// Faster version of the kernel, assuming that:
-// --> 'm' and 'n' are multiples of WGS2
-// --> 'a_offset' is 0
-// --> 'a_ld' is a multiple of VW2
-// --> 'a_rotated' is 0
-// --> 'do_conjugate' is 0
-__attribute__((reqd_work_group_size(WGS2, 1, 1)))
-__kernel void XgemvFast(const int m, const int n, const real alpha, const real beta,
-                        const int a_rotated,
-                        const __global realVF* restrict agm, const int a_offset, const int a_ld,
-                        const __global real* restrict xgm, const int x_offset, const int x_inc,
-                        __global real* ygm, const int y_offset, const int y_inc,
-                        const int do_conjugate) {
-  // Local memory for the vector X
-  __local real xlm[WGS2];
-
-  // Initializes the accumulation register
-  real acc[WPT2];
-  #pragma unroll
-  for (int w=0; w<WPT2; ++w) {
-    SetToZero(acc[w]);
-  }
-
-  // Loops over work-group sized portions of the work
-  for (int kwg=0; kwg<n; kwg+=WGS2) {
-
-    // Loads the vector X into local memory
-    const int lid = get_local_id(0);
-    xlm[lid] = xgm[(kwg + lid)*x_inc + x_offset];
-
-    // Synchronizes all threads in a workgroup
-    barrier(CLK_LOCAL_MEM_FENCE);
-
-    // The multiply-add function (not rotated)
-    #pragma unroll
-    for (int kl=0; kl<WGS2; ++kl) {
-      const int k = kwg + kl;
-      #pragma unroll
-      for (int w=0; w<WPT2/VW2; ++w) {
-        const int gid = (WPT2/VW2)*get_global_id(0) + w;
-        realVF avec = LoadMatrixAVF(agm, gid, k, a_ld/VW2);
-        #if VW2 == 1
-          MultiplyAdd(acc[VW2*w+0], xlm[kl], avec);
-        #elif VW2 == 2
-          MultiplyAdd(acc[VW2*w+0], xlm[kl], avec.x);
-          MultiplyAdd(acc[VW2*w+1], xlm[kl], avec.y);
-        #elif VW2 == 4
-          MultiplyAdd(acc[VW2*w+0], xlm[kl], avec.x);
-          MultiplyAdd(acc[VW2*w+1], xlm[kl], avec.y);
-          MultiplyAdd(acc[VW2*w+2], xlm[kl], avec.z);
-          MultiplyAdd(acc[VW2*w+3], xlm[kl], avec.w);
-        #elif VW2 == 8
-          MultiplyAdd(acc[VW2*w+0], xlm[kl], avec.s0);
-          MultiplyAdd(acc[VW2*w+1], xlm[kl], avec.s1);
-          MultiplyAdd(acc[VW2*w+2], xlm[kl], avec.s2);
-          MultiplyAdd(acc[VW2*w+3], xlm[kl], avec.s3);
-          MultiplyAdd(acc[VW2*w+4], xlm[kl], avec.s4);
-          MultiplyAdd(acc[VW2*w+5], xlm[kl], avec.s5);
-          MultiplyAdd(acc[VW2*w+6], xlm[kl], avec.s6);
-          MultiplyAdd(acc[VW2*w+7], xlm[kl], avec.s7);
-        #elif VW2 == 16
-          MultiplyAdd(acc[VW2*w+0], xlm[kl], avec.s0);
-          MultiplyAdd(acc[VW2*w+1], xlm[kl], avec.s1);
-          MultiplyAdd(acc[VW2*w+2], xlm[kl], avec.s2);
-          MultiplyAdd(acc[VW2*w+3], xlm[kl], avec.s3);
-          MultiplyAdd(acc[VW2*w+4], xlm[kl], avec.s4);
-          MultiplyAdd(acc[VW2*w+5], xlm[kl], avec.s5);
-          MultiplyAdd(acc[VW2*w+6], xlm[kl], avec.s6);
-          MultiplyAdd(acc[VW2*w+7], xlm[kl], avec.s7);
-          MultiplyAdd(acc[VW2*w+8], xlm[kl], avec.s8);
-          MultiplyAdd(acc[VW2*w+9], xlm[kl], avec.s9);
-          MultiplyAdd(acc[VW2*w+10], xlm[kl], avec.sA);
-          MultiplyAdd(acc[VW2*w+11], xlm[kl], avec.sB);
-          MultiplyAdd(acc[VW2*w+12], xlm[kl], avec.sC);
-          MultiplyAdd(acc[VW2*w+13], xlm[kl], avec.sD);
-          MultiplyAdd(acc[VW2*w+14], xlm[kl], avec.sE);
-          MultiplyAdd(acc[VW2*w+15], xlm[kl], avec.sF);
-        #endif
-      }
-    }
-
-    // Synchronizes all threads in a workgroup
-    barrier(CLK_LOCAL_MEM_FENCE);
-  }
-
-  // Stores the final result
-  #pragma unroll
-  for (int w=0; w<WPT2; ++w) {
-    const int gid = WPT2*get_global_id(0) + w;
-    real yval = ygm[gid*y_inc + y_offset];
-    AXPBY(ygm[gid*y_inc + y_offset], alpha, acc[w], beta, yval);
-  }
-}
-
-// =================================================================================================
-
-// Faster version of the kernel, assuming that:
-// --> 'm' and 'n' are multiples of WGS3
-// --> 'a_offset' is 0
-// --> 'a_ld' is a multiple of VW3
-// --> 'a_rotated' is 1
-// --> 'do_conjugate' is 0
-__attribute__((reqd_work_group_size(WGS3, 1, 1)))
-__kernel void XgemvFastRot(const int m, const int n, const real alpha, const real beta,
-                           const int a_rotated,
-                           const __global realVFR* restrict agm, const int a_offset, const int a_ld,
-                           const __global real* restrict xgm, const int x_offset, const int x_inc,
-                           __global real* ygm, const int y_offset, const int y_inc,
-                           const int do_conjugate) {
-  // Local memory for the vector X
-  __local real xlm[WGS3];
-
-  // Initializes the accumulation register
-  real acc[WPT3];
-  #pragma unroll
-  for (int w=0; w<WPT3; ++w) {
-    SetToZero(acc[w]);
-  }
-
-  // Loops over work-group sized portions of the work
-  for (int kwg=0; kwg<n; kwg+=WGS3) {
-
-    // Loads the vector X into local memory
-    const int lid = get_local_id(0);
-    xlm[lid] = xgm[(kwg + lid)*x_inc + x_offset];
-
-    // Synchronizes all threads in a workgroup
-    barrier(CLK_LOCAL_MEM_FENCE);
-
-    // The multiply-add function (rotated)
-    #pragma unroll
-    for (int kl=0; kl<WGS3/VW3; ++kl) {
-      const int k = (kwg/VW3) + kl;
-      #pragma unroll
-      for (int w=0; w<WPT3; ++w) {
-        const int gid = WPT3*get_global_id(0) + w;
-        realVFR avec = LoadMatrixAVFR(agm, k, gid, a_ld/VW3);
-        #if VW3 == 1
-          MultiplyAdd(acc[w], xlm[VW3*kl+0], avec);
-        #elif VW3 == 2
-          MultiplyAdd(acc[w], xlm[VW3*kl+0], avec.x);
-          MultiplyAdd(acc[w], xlm[VW3*kl+1], avec.y);
-        #elif VW3 == 4
-          MultiplyAdd(acc[w], xlm[VW3*kl+0], avec.x);
-          MultiplyAdd(acc[w], xlm[VW3*kl+1], avec.y);
-          MultiplyAdd(acc[w], xlm[VW3*kl+2], avec.z);
-          MultiplyAdd(acc[w], xlm[VW3*kl+3], avec.w);
-        #elif VW3 == 8
-          MultiplyAdd(acc[w], xlm[VW3*kl+0], avec.s0);
-          MultiplyAdd(acc[w], xlm[VW3*kl+1], avec.s1);
-          MultiplyAdd(acc[w], xlm[VW3*kl+2], avec.s2);
-          MultiplyAdd(acc[w], xlm[VW3*kl+3], avec.s3);
-          MultiplyAdd(acc[w], xlm[VW3*kl+4], avec.s4);
-          MultiplyAdd(acc[w], xlm[VW3*kl+5], avec.s5);
-          MultiplyAdd(acc[w], xlm[VW3*kl+6], avec.s6);
-          MultiplyAdd(acc[w], xlm[VW3*kl+7], avec.s7);
-        #elif VW3 == 16
-          MultiplyAdd(acc[w], xlm[VW3*kl+0], avec.s0);
-          MultiplyAdd(acc[w], xlm[VW3*kl+1], avec.s1);
-          MultiplyAdd(acc[w], xlm[VW3*kl+2], avec.s2);
-          MultiplyAdd(acc[w], xlm[VW3*kl+3], avec.s3);
-          MultiplyAdd(acc[w], xlm[VW3*kl+4], avec.s4);
-          MultiplyAdd(acc[w], xlm[VW3*kl+5], avec.s5);
-          MultiplyAdd(acc[w], xlm[VW3*kl+6], avec.s6);
-          MultiplyAdd(acc[w], xlm[VW3*kl+7], avec.s7);
-          MultiplyAdd(acc[w], xlm[VW3*kl+8], avec.s8);
-          MultiplyAdd(acc[w], xlm[VW3*kl+9], avec.s9);
-          MultiplyAdd(acc[w], xlm[VW3*kl+10], avec.sA);
-          MultiplyAdd(acc[w], xlm[VW3*kl+11], avec.sB);
-          MultiplyAdd(acc[w], xlm[VW3*kl+12], avec.sC);
-          MultiplyAdd(acc[w], xlm[VW3*kl+13], avec.sD);
-          MultiplyAdd(acc[w], xlm[VW3*kl+14], avec.sE);
-          MultiplyAdd(acc[w], xlm[VW3*kl+15], avec.sF);
-        #endif
-      }
-    }
-
-    // Synchronizes all threads in a workgroup
-    barrier(CLK_LOCAL_MEM_FENCE);
-  }
-
-  // Stores the final result
-  #pragma unroll
-  for (int w=0; w<WPT3; ++w) {
-    const int gid = WPT3*get_global_id(0) + w;
-    real yval = ygm[gid*y_inc + y_offset];
-    AXPBY(ygm[gid*y_inc + y_offset], alpha, acc[w], beta, yval);
-  }
-}
-
-// =================================================================================================
-
-// End of the C++11 raw string literal
-)"
-
-// =================================================================================================
diff --git a/src/routines/level2/xgbmv.cc b/src/routines/level2/xgbmv.cc
new file mode 100644
index 00000000..eac208b3
--- /dev/null
+++ b/src/routines/level2/xgbmv.cc
@@ -0,0 +1,117 @@
+
+// =================================================================================================
+// This file is part of the CLBlast project. The project is licensed under Apache Version 2.0. This
+// project loosely follows the Google C++ styleguide and uses a tab-size of two spaces and a max-
+// width of 100 characters per line.
+//
+// Author(s):
+//   Cedric Nugteren <www.cedricnugteren.nl>
+//
+// This file implements the Xgbmv class (see the header for information about the class).
+//
+// =================================================================================================
+
+#include "internal/routines/level2/xgbmv.h"
+
+#include <string>
+#include <vector>
+
+namespace clblast {
+// =================================================================================================
+
+// Constructor: forwards to base class constructor
+template <typename T>
+Xgbmv<T>::Xgbmv(Queue &queue, Event &event, const std::string &name):
+    Xgemv<T>(queue, event, name) {
+}
+
+// =================================================================================================
+
+// The main routine
+template <typename T>
+StatusCode Xgbmv<T>::DoGbmv(const Layout layout, const Transpose a_transpose,
+                            const size_t m, const size_t n, const size_t kl, const size_t ku,
+                            const T alpha,
+                            const Buffer<T> &a_buffer, const size_t a_offset, const size_t a_ld,
+                            const Buffer<T> &x_buffer, const size_t x_offset, const size_t x_inc,
+                            const T beta,
+                            const Buffer<T> &y_buffer, const size_t y_offset, const size_t y_inc) {
+
+  // Makes sure all dimensions are larger than zero
+  if (n == 0 || m == 0) { return StatusCode::kInvalidDimension; }
+
+  //
+  auto rotated = (layout == Layout::kRowMajor);
+  auto t_one = (rotated) ? n : m;
+  auto t_two = (rotated) ? m : n;
+  auto a_one = kl+ku+1;
+  auto a_two = (rotated) ? m : n;
+
+  // Checks for validity of the A matrix
+  auto status = StatusCode::kSuccess;
+  if (a_ld < a_one) { return StatusCode::kInvalidLeadDimA; }
+  try {
+    auto required_size = (a_ld*a_two + a_offset)*sizeof(T);
+    auto buffer_size = a_buffer.GetSize();
+    if (buffer_size < required_size) { return StatusCode::kInsufficientMemoryA; }
+  } catch (...) { return StatusCode::kInvalidMatrixA; }
+
+  // Temporary buffer to generalize the input matrix
+  try {
+    auto t_buffer = Buffer<T>(context_, t_one*t_two);
+
+    // Creates a general matrix from the input to be able to run the regular Xgemv routine
+    try {
+      auto& program = GetProgramFromCache();
+      auto kernel = Kernel(program, "GeneralBandedToGeneral");
+
+      // Sets the arguments for the matrix transform kernel
+      kernel.SetArgument(0, static_cast<int>(a_one));
+      kernel.SetArgument(1, static_cast<int>(a_two));
+      kernel.SetArgument(2, static_cast<int>(a_ld));
+      kernel.SetArgument(3, static_cast<int>(a_offset));
+      kernel.SetArgument(4, a_buffer());
+      kernel.SetArgument(5, static_cast<int>(t_one));
+      kernel.SetArgument(6, static_cast<int>(t_two));
+      kernel.SetArgument(7, static_cast<int>(t_one));
+      kernel.SetArgument(8, static_cast<int>(0));
+      kernel.SetArgument(9, t_buffer());
+      kernel.SetArgument(10, static_cast<int>(layout));
+      if (rotated) {
+        kernel.SetArgument(11, static_cast<int>(ku));
+        kernel.SetArgument(12, static_cast<int>(kl));
+      }
+      else {
+        kernel.SetArgument(11, static_cast<int>(kl));
+        kernel.SetArgument(12, static_cast<int>(ku));
+      }
+
+      // Uses the common matrix-transforms thread configuration
+      auto global = std::vector<size_t>{Ceil(CeilDiv(t_one, db_["PAD_WPTX"]), db_["PAD_DIMX"]),
+                                        Ceil(CeilDiv(t_two, db_["PAD_WPTY"]), db_["PAD_DIMY"])};
+      auto local = std::vector<size_t>{db_["PAD_DIMX"], db_["PAD_DIMY"]};
+      status = RunKernel(kernel, global, local);
+      if (ErrorIn(status)) { return status; }
+
+      // Runs the regular Xgemv code
+      status = DoGemv(layout, a_transpose, m, n, alpha,
+                      t_buffer, 0, t_one,
+                      x_buffer, x_offset, x_inc, beta,
+                      y_buffer, y_offset, y_inc);
+
+      // Return the status of the Xgemv routine
+      return status;
+    } catch (...) { return StatusCode::kInvalidKernel; }
+  } catch (...) { return StatusCode::kTempBufferAllocFailure; }
+}
+
+// =================================================================================================
+
+// Compiles the templated class
+template class Xgbmv<float>;
+template class Xgbmv<double>;
+template class Xgbmv<float2>;
+template class Xgbmv<double2>;
+
+// =================================================================================================
+} // namespace clblast
diff --git a/src/routines/level2/xgemv.cc b/src/routines/level2/xgemv.cc
index f95a9957..e52d2f20 100644
--- a/src/routines/level2/xgemv.cc
+++ b/src/routines/level2/xgemv.cc
@@ -32,8 +32,10 @@ template <typename T>
 Xgemv<T>::Xgemv(Queue &queue, Event &event, const std::string &name):
     Routine<T>(queue, event, name, {"Pad", "Xgemv"}, precision_) {
   source_string_ =
-    #include "../../kernels/pad.opencl" // For {Herm,Symm}{Upper,Lower}ToSquared (for HEMV/SYMV)
-    #include "../../kernels/xgemv.opencl"
+    #include "../../kernels/pad.opencl" // TODO: replace
+    #include "../../kernels/matrix_transforms/transforms.opencl"
+    #include "../../kernels/matrix_transforms/gbgemt.opencl"
+    #include "../../kernels/level2/xgemv.opencl"
   ;
 }
 
diff --git a/src/tuning/xgemv.cc b/src/tuning/xgemv.cc
index 3d6fe595..6a066518 100644
--- a/src/tuning/xgemv.cc
+++ b/src/tuning/xgemv.cc
@@ -34,7 +34,7 @@ class TuneXgemv {
   static std::string GetSources() {
     return
       #include "../src/kernels/common.opencl"
-      #include "../src/kernels/xgemv.opencl"
+      #include "../src/kernels/level2/xgemv.opencl"
     ;
   }
 
-- 
cgit v1.2.3