From 8833ae51be466686c7b5950ac8b5e598ce7012d0 Mon Sep 17 00:00:00 2001 From: Cedric Nugteren Date: Fri, 14 Apr 2017 20:16:51 +0200 Subject: Made the number of runs a benchmark-specific setting in the benchmark scripts --- scripts/benchmark/benchmark.py | 5 ++-- scripts/benchmark/settings.py | 64 +++++++++++++++++++++--------------------- 2 files changed, 35 insertions(+), 34 deletions(-) (limited to 'scripts/benchmark') diff --git a/scripts/benchmark/benchmark.py b/scripts/benchmark/benchmark.py index 31aa8c4f..85323fd1 100644 --- a/scripts/benchmark/benchmark.py +++ b/scripts/benchmark/benchmark.py @@ -67,7 +67,7 @@ def main(argv): parser.add_argument("-b", "--benchmark", required=True, help="The benchmark to perform (choose from %s)" % EXPERIMENTS.keys()) parser.add_argument("-p", "--platform", required=True, type=int, help="The ID of the OpenCL platform to test on") parser.add_argument("-d", "--device", required=True, type=int, help="The ID of the OpenCL device to test on") - parser.add_argument("-n", "--num_runs", type=int, default=10, help="The number of benchmark repeats for averaging") + parser.add_argument("-n", "--num_runs", type=int, default=None, help="Overrides the default number of benchmark repeats for averaging") parser.add_argument("-x", "--precision", type=int, default=32, help="The precision to test for (choose from 16, 32, 64, 3232, 6464") parser.add_argument("-l", "--load_from_disk", action="store_true", help="Increase verbosity of the script") @@ -102,8 +102,9 @@ def main(argv): results = {"label_names": experiment["label_names"], "num_rows": experiment["num_rows"], "num_cols": experiment["num_cols"], "benchmarks": []} for benchmark in benchmarks: + num_runs = benchmark["num_runs"] if cl_args.num_runs is None else cl_args.num_runs print("[benchmark] Running benchmark '%s:%s'" % (benchmark["name"], benchmark["title"])) - result = run_benchmark(benchmark["name"], benchmark["arguments"], cl_args.precision, cl_args.num_runs, + result = run_benchmark(benchmark["name"], benchmark["arguments"], cl_args.precision, num_runs, cl_args.platform, cl_args.device) results["benchmarks"].append(result) diff --git a/scripts/benchmark/settings.py b/scripts/benchmark/settings.py index cc7220e1..4b84d6b0 100644 --- a/scripts/benchmark/settings.py +++ b/scripts/benchmark/settings.py @@ -14,35 +14,35 @@ AXPY = { "num_rows": 2, "num_cols": 3, "benchmarks": [ { - "name": "axpy", + "name": "axpy", "num_runs": 40, "title": "multiples of 256K", "x_label": "sizes (n)", "x_keys": ["n"], "y_label": "GB/s (higher is better)", "y_keys": ["GBs_1", "GBs_2"], "arguments": [{"n": utils.k(256), "incx": 1, "incy": 1, "step": utils.k(256), "num_steps": 16}], }, { - "name": "axpy", + "name": "axpy", "num_runs": 40, "title": "multiples of 256K+1", "x_label": "sizes (n)", "x_keys": ["n"], "y_label": "GB/s (higher is better)", "y_keys": ["GBs_1", "GBs_2"], "arguments": [{"n": utils.k(256) + 1, "incx": 1, "incy": 1, "step": utils.k(256) + 1, "num_steps": 16}], }, { - "name": "axpy", + "name": "axpy", "num_runs": 40, "title": "around 1M", "x_label": "sizes (n)", "x_keys": ["n"], "y_label": "GB/s (higher is better)", "y_keys": ["GBs_1", "GBs_2"], "arguments": [{"n": utils.m(1), "incx": 1, "incy": 1, "step": 1, "num_steps": 16}], }, { - "name": "axpy", + "name": "axpy", "num_runs": 40, "title": "around 16M", "x_label": "sizes (n)", "x_keys": ["n"], "y_label": "GB/s (higher is better)", "y_keys": ["GBs_1", "GBs_2"], "arguments": [{"n": utils.m(16), "incx": 1, "incy": 1, "step": 1, "num_steps": 16}], }, { - "name": "axpy", + "name": "axpy", "num_runs": 40, "title": "strides n=8M", "x_label": "increments for x,y", "x_keys": ["incx", "incy"], "y_label": "GB/s (higher is better)", "y_keys": ["GBs_1", "GBs_2"], @@ -50,7 +50,7 @@ AXPY = { for inc_x in [1, 2, 4] for inc_y in [1, 2, 4]], }, { - "name": "axpy", + "name": "axpy", "num_runs": 40, "title": "powers of 2", "x_label": "sizes (n)", "x_keys": ["n"], "y_label": "GB/s (higher is better)", "y_keys": ["GBs_1", "GBs_2"], @@ -65,42 +65,42 @@ GEMV = { "num_rows": 2, "num_cols": 3, "benchmarks": [ { - "name": "gemv", + "name": "gemv", "num_runs": 10, "title": "multiples of 256", "x_label": "sizes (n=m)", "x_keys": ["n"], "y_label": "GB/s (higher is better)", "y_keys": ["GBs_1", "GBs_2"], "arguments": [{"n": 256, "m": 256, "incx": 1, "incy": 1, "layout": 102, "step": 256, "num_steps": 20}], }, { - "name": "gemv", + "name": "gemv", "num_runs": 10, "title": "multiples of 257", "x_label": "sizes (n=m)", "x_keys": ["n"], "y_label": "GB/s (higher is better)", "y_keys": ["GBs_1", "GBs_2"], "arguments": [{"n": 257, "m": 257, "incx": 1, "incy": 1, "layout": 102, "step": 257, "num_steps": 20}], }, { - "name": "gemv", + "name": "gemv", "num_runs": 10, "title": "around 4K", "x_label": "sizes (n=m)", "x_keys": ["n"], "y_label": "GB/s (higher is better)", "y_keys": ["GBs_1", "GBs_2"], "arguments": [{"n": 4096, "m": 4096, "incx": 1, "incy": 1, "layout": 102, "step": 1, "num_steps": 16}], }, { - "name": "gemv", + "name": "gemv", "num_runs": 10, "title": "multiples of 256 rotated", "x_label": "sizes (n=m)", "x_keys": ["n"], "y_label": "GB/s (higher is better)", "y_keys": ["GBs_1", "GBs_2"], "arguments": [{"n": 256, "m": 256, "incx": 1, "incy": 1, "layout": 101, "step": 256, "num_steps": 20}], }, { - "name": "gemv", + "name": "gemv", "num_runs": 10, "title": "multiples of 257 rotated", "x_label": "sizes (n=m)", "x_keys": ["n"], "y_label": "GB/s (higher is better)", "y_keys": ["GBs_1", "GBs_2"], "arguments": [{"n": 257, "m": 257, "incx": 1, "incy": 1, "layout": 101, "step": 257, "num_steps": 20}], }, { - "name": "gemv", + "name": "gemv", "num_runs": 10, "title": "strides n=m=4K", "x_label": "increments/strides for x,y", "x_keys": ["incx", "incy"], "y_label": "GB/s (higher is better)", "y_keys": ["GBs_1", "GBs_2"], @@ -115,7 +115,7 @@ GEMM = { "num_rows": 2, "num_cols": 3, "benchmarks": [ { - "name": "gemm", + "name": "gemm", "num_runs": 10, "title": "multiples of 128", "x_label": "sizes (m=n=k)", "x_keys": ["m"], "y_label": "GFLOPS (higher is better)", "y_keys": ["GFLOPS_1", "GFLOPS_2"], @@ -123,7 +123,7 @@ GEMM = { "transA": 111, "transB": 111, "step": 128, "num_steps": 20}], }, { - "name": "gemm", + "name": "gemm", "num_runs": 10, "title": "multiples of 129", "x_label": "sizes (m=n=k)", "x_keys": ["m"], "y_label": "GFLOPS (higher is better)", "y_keys": ["GFLOPS_1", "GFLOPS_2"], @@ -131,7 +131,7 @@ GEMM = { "transA": 111, "transB": 111, "step": 129, "num_steps": 20}], }, { - "name": "gemm", + "name": "gemm", "num_runs": 10, "title": "around 512", "x_label": "sizes (m=n=k)", "x_keys": ["m"], "y_label": "GFLOPS (higher is better)", "y_keys": ["GFLOPS_1", "GFLOPS_2"], @@ -139,7 +139,7 @@ GEMM = { "transA": 111, "transB": 111, "step": 1, "num_steps": 16}], }, { - "name": "gemm", + "name": "gemm", "num_runs": 10, "title": "around 2048", "x_label": "sizes (m=n=k)", "x_keys": ["m"], "y_label": "GFLOPS (higher is better)", "y_keys": ["GFLOPS_1", "GFLOPS_2"], @@ -147,7 +147,7 @@ GEMM = { "transA": 111, "transB": 111, "step": 1, "num_steps": 16}], }, { - "name": "gemm", + "name": "gemm", "num_runs": 10, "title": "layouts/transpose", "x_label": "layout, transA, transB", "x_keys": ["layout", "transA", "transB"], "y_label": "GFLOPS (higher is better)", "y_keys": ["GFLOPS_1", "GFLOPS_2"], @@ -156,7 +156,7 @@ GEMM = { for layout in [101, 102] for transA in [111, 112] for transB in [111, 112]], }, { - "name": "gemm", + "name": "gemm", "num_runs": 10, "title": "powers of 2", "x_label": "sizes (m=n=k)", "x_keys": ["m"], "y_label": "GFLOPS (higher is better)", "y_keys": ["GFLOPS_1", "GFLOPS_2"], @@ -172,7 +172,7 @@ GEMM_SMALL = { "num_rows": 2, "num_cols": 1, "benchmarks": [ { - "name": "gemm", + "name": "gemm", "num_runs": 10, "title": "small matrices in steps of 16", "x_label": "sizes (m=n=k)", "x_keys": ["m"], "y_label": "GFLOPS (higher is better)", "y_keys": ["GFLOPS_1", "GFLOPS_2"], @@ -180,7 +180,7 @@ GEMM_SMALL = { "transA": 111, "transB": 111, "step": 16, "num_steps": 57}], }, { - "name": "gemm", + "name": "gemm", "num_runs": 10, "title": "small matrices in steps of 1", "x_label": "sizes (m=n=k)", "x_keys": ["m"], "y_label": "GFLOPS (higher is better)", "y_keys": ["GFLOPS_1", "GFLOPS_2"], @@ -196,7 +196,7 @@ SYMM = { "num_rows": 2, "num_cols": 3, "benchmarks": [ { - "name": "symm", + "name": "symm", "num_runs": 10, "title": "multiples of 128", "x_label": "sizes (m=n)", "x_keys": ["m"], "y_label": "GFLOPS (higher is better)", "y_keys": ["GFLOPS_1", "GFLOPS_2"], @@ -204,7 +204,7 @@ SYMM = { "side": 141, "triangle": 121, "step": 128, "num_steps": 20}], }, { - "name": "symm", + "name": "symm", "num_runs": 10, "title": "multiples of 129", "x_label": "sizes (m=n)", "x_keys": ["m"], "y_label": "GFLOPS (higher is better)", "y_keys": ["GFLOPS_1", "GFLOPS_2"], @@ -212,7 +212,7 @@ SYMM = { "side": 141, "triangle": 121, "step": 129, "num_steps": 20}], }, { - "name": "symm", + "name": "symm", "num_runs": 10, "title": "around 512", "x_label": "sizes (m=n)", "x_keys": ["m"], "y_label": "GFLOPS (higher is better)", "y_keys": ["GFLOPS_1", "GFLOPS_2"], @@ -220,7 +220,7 @@ SYMM = { "side": 141, "triangle": 121, "step": 1, "num_steps": 16}], }, { - "name": "symm", + "name": "symm", "num_runs": 10, "title": "around 2048", "x_label": "sizes (m=n)", "x_keys": ["m"], "y_label": "GFLOPS (higher is better)", "y_keys": ["GFLOPS_1", "GFLOPS_2"], @@ -228,7 +228,7 @@ SYMM = { "side": 141, "triangle": 121, "step": 1, "num_steps": 16}], }, { - "name": "symm", + "name": "symm", "num_runs": 10, "title": "layouts/sides/triangles", "x_label": "layout, side, triangle", "x_keys": ["layout", "side", "triangle"], "y_label": "GFLOPS (higher is better)", "y_keys": ["GFLOPS_1", "GFLOPS_2"], @@ -237,7 +237,7 @@ SYMM = { for layout in [101, 102] for side in [141, 142] for triangle in [121, 122]], }, { - "name": "symm", + "name": "symm", "num_runs": 10, "title": "powers of 2", "x_label": "sizes (m=n)", "x_keys": ["m"], "y_label": "GFLOPS (higher is better)", "y_keys": ["GFLOPS_1", "GFLOPS_2"], @@ -253,7 +253,7 @@ SYRK = { "num_rows": 2, "num_cols": 3, "benchmarks": [ { - "name": "syrk", + "name": "syrk", "num_runs": 10, "title": "multiples of 128", "x_label": "sizes (n=k)", "x_keys": ["n"], "y_label": "GFLOPS (higher is better)", "y_keys": ["GFLOPS_1", "GFLOPS_2"], @@ -261,7 +261,7 @@ SYRK = { "side": 141, "triangle": 121, "step": 128, "num_steps": 20}], }, { - "name": "syrk", + "name": "syrk", "num_runs": 10, "title": "multiples of 129", "x_label": "sizes (n=k)", "x_keys": ["n"], "y_label": "GFLOPS (higher is better)", "y_keys": ["GFLOPS_1", "GFLOPS_2"], @@ -269,7 +269,7 @@ SYRK = { "side": 141, "triangle": 121, "step": 129, "num_steps": 20}], }, { - "name": "syrk", + "name": "syrk", "num_runs": 10, "title": "around 512", "x_label": "sizes (n=k)", "x_keys": ["n"], "y_label": "GFLOPS (higher is better)", "y_keys": ["GFLOPS_1", "GFLOPS_2"], @@ -277,7 +277,7 @@ SYRK = { "side": 141, "triangle": 121, "step": 1, "num_steps": 16}], }, { - "name": "syrk", + "name": "syrk", "num_runs": 10, "title": "around 2048", "x_label": "sizes (n=k)", "x_keys": ["n"], "y_label": "GFLOPS (higher is better)", "y_keys": ["GFLOPS_1", "GFLOPS_2"], @@ -285,7 +285,7 @@ SYRK = { "side": 141, "triangle": 121, "step": 1, "num_steps": 16}], }, { - "name": "syrk", + "name": "syrk", "num_runs": 10, "title": "layouts/sides/triangles", "x_label": "layout, triangle, transA", "x_keys": ["layout", "triangle", "transA"], "y_label": "GFLOPS (higher is better)", "y_keys": ["GFLOPS_1", "GFLOPS_2"], @@ -294,7 +294,7 @@ SYRK = { for layout in [101, 102] for triangle in [121, 122] for transA in [111, 112]], }, { - "name": "syrk", + "name": "syrk", "num_runs": 10, "title": "powers of 2", "x_label": "sizes (n=k)", "x_keys": ["n"], "y_label": "GFLOPS (higher is better)", "y_keys": ["GFLOPS_1", "GFLOPS_2"], -- cgit v1.2.3