1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
|
// =================================================================================================
// This file is part of the CLBlast project. The project is licensed under Apache Version 2.0. This
// project loosely follows the Google C++ styleguide and uses a tab-size of two spaces and a max-
// width of 100 characters per line.
//
// Author(s):
// Cedric Nugteren <www.cedricnugteren.nl>
//
// This file implements the Tester class, providing a test-framework. GTest was used before, but
// was not able to handle certain cases (e.g. template type + parameters). This is its (basic)
// custom replacement.
// Typename T: the data-type of the routine's memory buffers (==precision)
// Typename U: the data-type of the alpha and beta arguments
//
// =================================================================================================
#ifndef CLBLAST_TEST_CORRECTNESS_TESTER_H_
#define CLBLAST_TEST_CORRECTNESS_TESTER_H_
#include <string>
#include <vector>
#include <memory>
#include "utilities/utilities.hpp"
#include "test/test_utilities.hpp"
// The libraries
#ifdef CLBLAST_REF_CLBLAS
#include <clBLAS.h>
#endif
namespace clblast {
// =================================================================================================
// See comment at top of file for a description of the class
template <typename T, typename U>
class Tester {
public:
// Maximum number of test results printed on a single line
static const size_t kResultsPerLine;
// Error percentage is not applicable: error was caused by an incorrect status
static const float kStatusError;
// Constants holding start and end strings for terminal-output in colour
static const std::string kPrintError;
static const std::string kPrintSuccess;
static const std::string kPrintWarning;
static const std::string kPrintMessage;
static const std::string kPrintEnd;
// Sets the output error coding
static const std::string kSuccessData;
static const std::string kSuccessStatus;
static const std::string kErrorData;
static const std::string kErrorStatus;
static const std::string kSkippedCompilation;
static const std::string kUnsupportedPrecision;
static const std::string kUnsupportedReference;
// This structure combines the above log-entry with a status code an error percentage
struct ErrorLogEntry {
StatusCode status_expect;
StatusCode status_found;
float error_percentage;
Arguments<U> args;
};
// Creates an instance of the tester, running on a particular OpenCL platform and device. It
// takes the routine's names as an additional parameter.
explicit Tester(const std::vector<std::string> &arguments, const bool silent,
const std::string &name, const std::vector<std::string> &options);
~Tester();
// These methods start and end a test-case. Within a test-case, multiple tests can be run.
void TestStart(const std::string &test_name, const std::string &test_configuration);
void TestEnd();
// Tests either an error count (should be zero) or two error codes (must match)
void TestErrorCount(const size_t errors, const size_t size, const Arguments<U> &args);
void TestErrorCodes(const StatusCode clblas_status, const StatusCode clblast_status,
const Arguments<U> &args);
// Returns the number of failed tests
size_t NumFailedTests() const { return tests_failed_; }
protected:
// The help-message
std::string help_;
// The OpenCL objects (accessible by derived classes)
Platform platform_;
Device device_;
Context context_;
Queue queue_;
// Whether or not to run the full test-suite or just a smoke test
const bool full_test_;
// Whether or not to print extra information when testing
const bool verbose_;
// Retrieves the offset values to test with
const std::vector<size_t> GetOffsets() const;
// Retrieves the list of options as a string
std::string GetOptionsString(const Arguments<U> &args); // for regular tests
std::string GetSizesString(const Arguments<U> &args); // for invalid buffer sizes
// Testing against reference implementations
int compare_cblas_;
int compare_clblas_;
int compare_cublas_;
private:
// Internal methods to report a passed, skipped, or failed test
void ReportPass();
void ReportSkipped();
void ReportError(const ErrorLogEntry &log_entry);
// Prints the error or success symbol to screen
void PrintTestResult(const std::string &message);
// Prints an error log
void PrintErrorLog(const std::vector<ErrorLogEntry> &error_log);
// Logging and counting occurrences of errors
std::vector<ErrorLogEntry> error_log_;
size_t num_passed_;
size_t num_skipped_;
size_t num_failed_;
// Counting the amount of errors printed on this row
size_t print_count_;
// Counting the number of test-cases with and without failures
size_t tests_passed_;
size_t tests_skipped_;
size_t tests_failed_;
// Arguments relevant for a specific routine
std::vector<std::string> options_;
};
// Maximum number of test results printed on a single line
template <typename T, typename U> const size_t Tester<T,U>::kResultsPerLine = size_t{64};
// Error percentage is not applicable: error was caused by an incorrect status
template <typename T, typename U> const float Tester<T,U>::kStatusError = -1.0f;
// Constants holding start and end strings for terminal-output in colour
#if defined(_WIN32)
template <typename T, typename U> const std::string Tester<T,U>::kPrintError = "";
template <typename T, typename U> const std::string Tester<T,U>::kPrintSuccess = "";
template <typename T, typename U> const std::string Tester<T,U>::kPrintWarning = "";
template <typename T, typename U> const std::string Tester<T,U>::kPrintMessage = "";
template <typename T, typename U> const std::string Tester<T,U>::kPrintEnd = "";
#else
template <typename T, typename U> const std::string Tester<T,U>::kPrintError = "\x1b[31m";
template <typename T, typename U> const std::string Tester<T,U>::kPrintSuccess = "\x1b[32m";
template <typename T, typename U> const std::string Tester<T,U>::kPrintWarning = "\x1b[35m";
template <typename T, typename U> const std::string Tester<T,U>::kPrintMessage = "\x1b[1m";
template <typename T, typename U> const std::string Tester<T,U>::kPrintEnd = "\x1b[0m";
#endif
// Sets the output error coding
#if defined(_WIN32)
template <typename T, typename U> const std::string Tester<T,U>::kSuccessData = ":"; // success
template <typename T, typename U> const std::string Tester<T,U>::kSuccessStatus = "."; // success
template <typename T, typename U> const std::string Tester<T,U>::kErrorData = "X"; // error
template <typename T, typename U> const std::string Tester<T,U>::kErrorStatus = "/"; // error
template <typename T, typename U> const std::string Tester<T,U>::kSkippedCompilation = "\\"; // warning
template <typename T, typename U> const std::string Tester<T,U>::kUnsupportedPrecision = "o"; // warning
template <typename T, typename U> const std::string Tester<T,U>::kUnsupportedReference = "-"; // warning
#else
template <typename T, typename U> const std::string Tester<T,U>::kSuccessData = "\x1b[32m:\x1b[0m"; // success
template <typename T, typename U> const std::string Tester<T,U>::kSuccessStatus = "\x1b[32m.\x1b[0m"; // success
template <typename T, typename U> const std::string Tester<T,U>::kErrorData = "\x1b[31mX\x1b[0m"; // error
template <typename T, typename U> const std::string Tester<T,U>::kErrorStatus = "\x1b[31m/\x1b[0m"; // error
template <typename T, typename U> const std::string Tester<T,U>::kSkippedCompilation = "\x1b[35m\\\x1b[0m"; // warning
template <typename T, typename U> const std::string Tester<T,U>::kUnsupportedPrecision = "\x1b[35mo\x1b[0m"; // warning
template <typename T, typename U> const std::string Tester<T,U>::kUnsupportedReference = "\x1b[35m-\x1b[0m"; // warning
#endif
// =================================================================================================
// Below are the non-member functions (separated because of otherwise required partial class
// template specialization)
// =================================================================================================
// Error margins
template <typename T> float getRelativeErrorMargin();
template <typename T> float getAbsoluteErrorMargin();
template <typename T> double getL2ErrorMargin();
// Compares two floating point values and returns whether they are within an acceptable error
// margin. This replaces GTest's EXPECT_NEAR().
template <typename T>
bool TestSimilarity(const T val1, const T val2);
// Retrieves a list of example scalar values, used for the alpha and beta arguments for the various
// routines. This function is specialised for the different data-types.
template <typename T>
const std::vector<T> GetExampleScalars(const bool full_test);
// =================================================================================================
} // namespace clblast
// CLBLAST_TEST_CORRECTNESS_TESTER_H_
#endif
|
