From 2b1e0295e6dba8d8d9f85ca65b6232a89e6cceae Mon Sep 17 00:00:00 2001 From: Cedric Nugteren Date: Tue, 24 Apr 2018 20:41:15 +0200 Subject: Added a define to enable subgroup shuffling if supported by the device --- src/utilities/compile.cpp | 5 +++++ src/utilities/utilities.hpp | 1 + 2 files changed, 6 insertions(+) (limited to 'src') diff --git a/src/utilities/compile.cpp b/src/utilities/compile.cpp index c1d921a4..65131cca 100644 --- a/src/utilities/compile.cpp +++ b/src/utilities/compile.cpp @@ -57,6 +57,11 @@ Program CompileFromSource(const std::string &source_string, const Precision prec header_string += "#define GLOBAL_MEM_FENCE 1\n"; } + // For Intel GPUs with subgroup support, use subgroup shuffling. + if (device.IsGPU() && device.HasExtension(kKhronosIntelSubgroups)) { + header_string += "#define USE_SUBGROUP_SHUFFLING 1\n"; + } + // Optionally adds a translation header from OpenCL kernels to CUDA kernels #ifdef CUDA_API header_string += diff --git a/src/utilities/utilities.hpp b/src/utilities/utilities.hpp index d382b331..0edf77fe 100644 --- a/src/utilities/utilities.hpp +++ b/src/utilities/utilities.hpp @@ -47,6 +47,7 @@ using double2 = std::complex; // Khronos OpenCL extensions const std::string kKhronosAttributesAMD = "cl_amd_device_attribute_query"; const std::string kKhronosAttributesNVIDIA = "cl_nv_device_attribute_query"; +const std::string kKhronosIntelSubgroups = "cl_intel_subgroups"; // Catched an unknown error constexpr auto kUnknownError = -999; -- cgit v1.2.3 From 2965b87dda80ca22bf12527755ef9f3cb5fed46b Mon Sep 17 00:00:00 2001 From: Cedric Nugteren Date: Tue, 24 Apr 2018 21:32:42 +0200 Subject: Added Intel subgroup shuffle support to the 2D register caching GEMM kernel --- src/kernels/level3/xgemm_part1.opencl | 12 ++++++++++++ src/kernels/level3/xgemm_part3.opencl | 34 ++++++++++++++++++++++++++-------- 2 files changed, 38 insertions(+), 8 deletions(-) (limited to 'src') diff --git a/src/kernels/level3/xgemm_part1.opencl b/src/kernels/level3/xgemm_part1.opencl index d15dafc8..99d64c91 100644 --- a/src/kernels/level3/xgemm_part1.opencl +++ b/src/kernels/level3/xgemm_part1.opencl @@ -114,6 +114,18 @@ R"( #define GLOBAL_MEM_FENCE 0 // Global synchronisation barrier for potential better performance #endif +// Intel subgroups (https://www.khronos.org/registry/OpenCL/extensions/intel/cl_intel_subgroups.txt) +#ifndef USE_SUBGROUP_SHUFFLING + #define USE_SUBGROUP_SHUFFLING 0 // Optionally enables subgroup shuffling for Intel GPUs +#endif +#if USE_SUBGROUP_SHUFFLING == 1 + #define SUBGROUP_SIZE 8 // Assumes subgroup size is always 8 on Intel GPUs +#endif +#if NWI != SUBGROUP_SIZE || MDIMC < SUBGROUP_SIZE + #undef USE_SUBGROUP_SHUFFLING + #define USE_SUBGROUP_SHUFFLING 0 // Disables subgroups in case the assumptions don't hold +#endif + // ================================================================================================= // Data-widths in dimension M diff --git a/src/kernels/level3/xgemm_part3.opencl b/src/kernels/level3/xgemm_part3.opencl index c25c3001..c3920cb5 100644 --- a/src/kernels/level3/xgemm_part3.opencl +++ b/src/kernels/level3/xgemm_part3.opencl @@ -37,8 +37,13 @@ INLINE_FUNC void XgemmBody(const int kSizeM, const int kSizeN, const int kSizeK, #pragma promote_to_registers realN bpm[NWI/VWN]; // 1 * NWI #elif GEMMK == 1 - #pragma promote_to_registers - realN apm[NWI*(KREG/VWN)]; // NWI * KREG + #if USE_SUBGROUP_SHUFFLING == 1 + #pragma promote_to_registers + realN apm[KREG/VWN]; // KREG (subgroup shuffling in NWI dimension) + #else + #pragma promote_to_registers + realN apm[NWI*(KREG/VWN)]; // NWI * KREG + #endif #pragma promote_to_registers realM bpm[KREG*(MWI/VWM)]; // KREG * MWI #endif @@ -123,14 +128,23 @@ INLINE_FUNC void XgemmBody(const int kSizeM, const int kSizeN, const int kSizeK, #endif } #elif GEMMK == 1 - // Loads data: 2D global --> 2D private (matrix A) - #pragma unroll - for (int _ni = 0; _ni < NWI; _ni += 1) { + // Loads data: 2D global --> 2D private (matrix A). Partly, shuffled later among subgroups + #if USE_SUBGROUP_SHUFFLING == 1 + const int _ni = get_sub_group_local_id(); #pragma unroll for (int _ki = 0; _ki < KREG/VWN; _ki += 1) { - apm[_ni * (KREG/VWN) + _ki] = GlobalToPrivateA2D(a_ptr, tid_y, _ni, kSizeK, idk, _ki); + apm[_ki] = GlobalToPrivateA2D(a_ptr, tid_y, _ni, kSizeK, idk, _ki); } - } + // Loads data: 2D global --> 2D private (matrix A) + #else + #pragma unroll + for (int _ni = 0; _ni < NWI; _ni += 1) { + #pragma unroll + for (int _ki = 0; _ki < KREG/VWN; _ki += 1) { + apm[_ni * (KREG/VWN) + _ki] = GlobalToPrivateA2D(a_ptr, tid_y, _ni, kSizeK, idk, _ki); + } + } + #endif #endif // Performs the accumulation (Cpm += Apm * Bpm) @@ -187,7 +201,11 @@ INLINE_FUNC void XgemmBody(const int kSizeM, const int kSizeN, const int kSizeK, #pragma unroll for (int _ki = 0; _ki < KREG/VWN; _ki += 1) { const int index = _ni * (MWI/VWM) + _mi; - const realN aval = apm[_ni * (KREG/VWN) + _ki]; + #if USE_SUBGROUP_SHUFFLING == 1 + const realN aval = intel_sub_group_shuffle(apm[_ki], _ni); + #else + const realN aval = apm[_ni * (KREG/VWN) + _ki]; + #endif #if VWN == 1 cpm[index] = MultiplyAddVector(cpm[index], bpm[(VWN * _ki + 0) * (MWI/VWM) + _mi], aval); #elif VWN == 2 -- cgit v1.2.3