diff options
author | Cedric Nugteren <web@cedricnugteren.nl> | 2018-01-11 19:42:50 +0100 |
---|---|---|
committer | GitHub <noreply@github.com> | 2018-01-11 19:42:50 +0100 |
commit | 9b084d04093fdbfb22ee4790c6b3db5c55cd2719 (patch) | |
tree | d8f8bc1b3884c0340df9f6d95b4837ed3dff8deb /src/kernels/level3/xgemm_direct_batched.opencl | |
parent | c988c2cdd166ebf6d5b5ec20f445de1a95a65b16 (diff) | |
parent | 99a4df88a6d808ea77c9116ce63621503c00b57a (diff) |
Merge pull request #239 from CNugteren/gemm_strided_batched
GemmStridedBatched
Diffstat (limited to 'src/kernels/level3/xgemm_direct_batched.opencl')
-rw-r--r-- | src/kernels/level3/xgemm_direct_batched.opencl | 122 |
1 files changed, 102 insertions, 20 deletions
diff --git a/src/kernels/level3/xgemm_direct_batched.opencl b/src/kernels/level3/xgemm_direct_batched.opencl index d946a056..d15ed31e 100644 --- a/src/kernels/level3/xgemm_direct_batched.opencl +++ b/src/kernels/level3/xgemm_direct_batched.opencl @@ -17,15 +17,16 @@ R"( // ================================================================================================= +#if defined(ROUTINE_GEMMBATCHED) // Direct version of the batched GEMM kernel with [A, B] = [non-transposed, non-transposed] __kernel __attribute__((reqd_work_group_size(MDIMCD, NDIMCD, 1))) void XgemmDirectBatchedNN(const int kSizeM, const int kSizeN, const int kSizeK, - const __constant real_arg* arg_alphas, const __constant real_arg* arg_betas, - const __global realMD* restrict agm, const __constant int* a_offsets, const int a_ld, - const __global realND* restrict bgm, const __constant int* b_offsets, const int b_ld, - __global real* cgm, const __constant int* c_offsets, const int c_ld, - const int c_transpose, const int a_conjugate, const int b_conjugate) { + const __constant real_arg* arg_alphas, const __constant real_arg* arg_betas, + const __global realMD* restrict agm, const __constant int* a_offsets, const int a_ld, + const __global realND* restrict bgm, const __constant int* b_offsets, const int b_ld, + __global real* cgm, const __constant int* c_offsets, const int c_ld, + const int c_transpose, const int a_conjugate, const int b_conjugate) { const int batch = get_group_id(2); const real_arg arg_alpha = arg_alphas[batch]; const real_arg arg_beta = arg_betas[batch]; @@ -42,11 +43,11 @@ void XgemmDirectBatchedNN(const int kSizeM, const int kSizeN, const int kSizeK, // Direct version of the batched GEMM kernel with [A, B] = [non-transposed, transposed] __kernel __attribute__((reqd_work_group_size(MDIMCD, NDIMCD, 1))) void XgemmDirectBatchedNT(const int kSizeM, const int kSizeN, const int kSizeK, - const __constant real_arg* arg_alphas, const __constant real_arg* arg_betas, - const __global realMD* restrict agm, const __constant int* a_offsets, const int a_ld, - const __global realND* restrict bgm, const __constant int* b_offsets, const int b_ld, - __global real* cgm, const __constant int* c_offsets, const int c_ld, - const int c_transpose, const int a_conjugate, const int b_conjugate) { + const __constant real_arg* arg_alphas, const __constant real_arg* arg_betas, + const __global realMD* restrict agm, const __constant int* a_offsets, const int a_ld, + const __global realND* restrict bgm, const __constant int* b_offsets, const int b_ld, + __global real* cgm, const __constant int* c_offsets, const int c_ld, + const int c_transpose, const int a_conjugate, const int b_conjugate) { const int batch = get_group_id(2); const real_arg arg_alpha = arg_alphas[batch]; const real_arg arg_beta = arg_betas[batch]; @@ -63,11 +64,11 @@ void XgemmDirectBatchedNT(const int kSizeM, const int kSizeN, const int kSizeK, // Direct version of the batched GEMM kernel with [A, B] = [transposed, non-transposed] __kernel __attribute__((reqd_work_group_size(MDIMCD, NDIMCD, 1))) void XgemmDirectBatchedTN(const int kSizeM, const int kSizeN, const int kSizeK, - const __constant real_arg* arg_alphas, const __constant real_arg* arg_betas, - const __global realMD* restrict agm, const __constant int* a_offsets, const int a_ld, - const __global realND* restrict bgm, const __constant int* b_offsets, const int b_ld, - __global real* cgm, const __constant int* c_offsets, const int c_ld, - const int c_transpose, const int a_conjugate, const int b_conjugate) { + const __constant real_arg* arg_alphas, const __constant real_arg* arg_betas, + const __global realMD* restrict agm, const __constant int* a_offsets, const int a_ld, + const __global realND* restrict bgm, const __constant int* b_offsets, const int b_ld, + __global real* cgm, const __constant int* c_offsets, const int c_ld, + const int c_transpose, const int a_conjugate, const int b_conjugate) { const int batch = get_group_id(2); const real_arg arg_alpha = arg_alphas[batch]; const real_arg arg_beta = arg_betas[batch]; @@ -84,11 +85,11 @@ void XgemmDirectBatchedTN(const int kSizeM, const int kSizeN, const int kSizeK, // Direct version of the batched GEMM kernel with [A, B] = [transposed, transposed] __kernel __attribute__((reqd_work_group_size(MDIMCD, NDIMCD, 1))) void XgemmDirectBatchedTT(const int kSizeM, const int kSizeN, const int kSizeK, - const __constant real_arg* arg_alphas, const __constant real_arg* arg_betas, - const __global realMD* restrict agm, const __constant int* a_offsets, const int a_ld, - const __global realND* restrict bgm, const __constant int* b_offsets, const int b_ld, - __global real* cgm, const __constant int* c_offsets, const int c_ld, - const int c_transpose, const int a_conjugate, const int b_conjugate) { + const __constant real_arg* arg_alphas, const __constant real_arg* arg_betas, + const __global realMD* restrict agm, const __constant int* a_offsets, const int a_ld, + const __global realND* restrict bgm, const __constant int* b_offsets, const int b_ld, + __global real* cgm, const __constant int* c_offsets, const int c_ld, + const int c_transpose, const int a_conjugate, const int b_conjugate) { const int batch = get_group_id(2); const real_arg arg_alpha = arg_alphas[batch]; const real_arg arg_beta = arg_betas[batch]; @@ -102,6 +103,87 @@ void XgemmDirectBatchedTT(const int kSizeM, const int kSizeN, const int kSizeK, alm, blm, 1, 1, c_transpose, a_conjugate, b_conjugate); } +#endif +// ================================================================================================= +#if defined(ROUTINE_GEMMSTRIDEDBATCHED) + +// Direct version of the strided-batched GEMM kernel with [A, B] = [non-transposed, non-transposed] +__kernel __attribute__((reqd_work_group_size(MDIMCD, NDIMCD, 1))) +void XgemmDirectStridedBatchedNN(const int kSizeM, const int kSizeN, const int kSizeK, + const real_arg arg_alpha, const real_arg arg_beta, + const __global realMD* restrict agm, const int a_offset, const int a_ld, const int a_stride, + const __global realND* restrict bgm, const int b_offset, const int b_ld, const int b_stride, + __global real* cgm, const int c_offset, const int c_ld, const int c_stride, + const int c_transpose, const int a_conjugate, const int b_conjugate) { + const int batch = get_group_id(2); + const int a_offset_batch = a_offset + a_stride * batch; + const int b_offset_batch = b_offset + b_stride * batch; + const int c_offset_batch = c_offset + c_stride * batch; + __local real alm[WGD * (WGD + PADA)]; + __local real blm[WGD * (WGD + PADB)]; + XgemmDirect(kSizeM, kSizeN, kSizeK, arg_alpha, arg_beta, + agm, a_offset_batch, a_ld, bgm, b_offset_batch, b_ld, cgm, c_offset_batch, c_ld, + alm, blm, 0, 0, c_transpose, a_conjugate, b_conjugate); +} + +// Direct version of the strided-batched GEMM kernel with [A, B] = [non-transposed, transposed] +__kernel __attribute__((reqd_work_group_size(MDIMCD, NDIMCD, 1))) +void XgemmDirectStridedBatchedNT(const int kSizeM, const int kSizeN, const int kSizeK, + const real_arg arg_alpha, const real_arg arg_beta, + const __global realMD* restrict agm, const int a_offset, const int a_ld, const int a_stride, + const __global realND* restrict bgm, const int b_offset, const int b_ld, const int b_stride, + __global real* cgm, const int c_offset, const int c_ld, const int c_stride, + const int c_transpose, const int a_conjugate, const int b_conjugate) { + const int batch = get_group_id(2); + const int a_offset_batch = a_offset + a_stride * batch; + const int b_offset_batch = b_offset + b_stride * batch; + const int c_offset_batch = c_offset + c_stride * batch; + __local real alm[WGD * (WGD + PADA)]; + __local real blm[WGD * (WGD + PADB)]; + XgemmDirect(kSizeM, kSizeN, kSizeK, arg_alpha, arg_beta, + agm, a_offset_batch, a_ld, bgm, b_offset_batch, b_ld, cgm, c_offset_batch, c_ld, + alm, blm, 0, 1, c_transpose, a_conjugate, b_conjugate); +} + +// Direct version of the strided-batched GEMM kernel with [A, B] = [transposed, non-transposed] +__kernel __attribute__((reqd_work_group_size(MDIMCD, NDIMCD, 1))) +void XgemmDirectStridedBatchedTN(const int kSizeM, const int kSizeN, const int kSizeK, + const real_arg arg_alpha, const real_arg arg_beta, + const __global realMD* restrict agm, const int a_offset, const int a_ld, const int a_stride, + const __global realND* restrict bgm, const int b_offset, const int b_ld, const int b_stride, + __global real* cgm, const int c_offset, const int c_ld, const int c_stride, + const int c_transpose, const int a_conjugate, const int b_conjugate) { + const int batch = get_group_id(2); + const int a_offset_batch = a_offset + a_stride * batch; + const int b_offset_batch = b_offset + b_stride * batch; + const int c_offset_batch = c_offset + c_stride * batch; + __local real alm[WGD * (WGD + PADA)]; + __local real blm[WGD * (WGD + PADB)]; + XgemmDirect(kSizeM, kSizeN, kSizeK, arg_alpha, arg_beta, + agm, a_offset_batch, a_ld, bgm, b_offset_batch, b_ld, cgm, c_offset_batch, c_ld, + alm, blm, 1, 0, c_transpose, a_conjugate, b_conjugate); +} + +// Direct version of the strided-batched GEMM kernel with [A, B] = [transposed, transposed] +__kernel __attribute__((reqd_work_group_size(MDIMCD, NDIMCD, 1))) +void XgemmDirectStridedBatchedTT(const int kSizeM, const int kSizeN, const int kSizeK, + const real_arg arg_alpha, const real_arg arg_beta, + const __global realMD* restrict agm, const int a_offset, const int a_ld, const int a_stride, + const __global realND* restrict bgm, const int b_offset, const int b_ld, const int b_stride, + __global real* cgm, const int c_offset, const int c_ld, const int c_stride, + const int c_transpose, const int a_conjugate, const int b_conjugate) { + const int batch = get_group_id(2); + const int a_offset_batch = a_offset + a_stride * batch; + const int b_offset_batch = b_offset + b_stride * batch; + const int c_offset_batch = c_offset + c_stride * batch; + __local real alm[WGD * (WGD + PADA)]; + __local real blm[WGD * (WGD + PADB)]; + XgemmDirect(kSizeM, kSizeN, kSizeK, arg_alpha, arg_beta, + agm, a_offset_batch, a_ld, bgm, b_offset_batch, b_ld, cgm, c_offset_batch, c_ld, + alm, blm, 1, 1, c_transpose, a_conjugate, b_conjugate); +} + +#endif // ================================================================================================= // End of the C++11 raw string literal |