From c3d0e3f3d0af0a21f38647477293f5ee8c4373a6 Mon Sep 17 00:00:00 2001
From: Marcel Maltry <mmaltry+github@mailbox.org>
Date: Wed, 21 Feb 2024 10:53:39 +0100
Subject: [PATCH] [Benchmark] Add CLI argument `num_runs`

Previously, the number of runs had to be adjusted by changing the
corresponding global variable in the script.  This can now be done via
CLI argument.  The default is still 5 runs per system, case, and
configuration.
---
 benchmark/Benchmark.py | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/benchmark/Benchmark.py b/benchmark/Benchmark.py
index 4af978c2..c1533b60 100755
--- a/benchmark/Benchmark.py
+++ b/benchmark/Benchmark.py
@@ -26,8 +26,6 @@
 
 BENCHMARK_SYSTEMS: list[str] = ['mutable', 'PostgreSQL', 'DuckDB', 'HyPer']     # List of systems
 
-N_RUNS: int = 5
-
 
 class BenchmarkError(Exception):
     pass
@@ -263,7 +261,8 @@ def perform_experiment(
     system: str,
     info: SimpleNamespace,
     results: Result,
-    output_csv_file: str | None
+    output_csv_file: str | None,
+    num_runs: int
 ) -> None:
     # Experiment parameters
     systems: dict[str, Any] = yml.get('systems', dict())
@@ -279,7 +278,7 @@ def perform_experiment(
 
     # Perform benchmark
     try:
-        connector_result: ConnectorResult = conn.execute(N_RUNS, params)
+        connector_result: ConnectorResult = conn.execute(num_runs, params)
     except connector.ConnectorException as ex:
         tqdm_print(f"\nAn error occurred for {system} while executing {info.path_to_file}: {str(ex)}\n")
         raise BenchmarkError()
@@ -479,7 +478,7 @@ def run_benchmarks(args: argparse.Namespace) -> None:
                     if system == 'mutable':
                         num_experiments_total += 1
                     try:
-                        perform_experiment(yml, conn, system, info, results, output_csv_file)
+                        perform_experiment(yml, conn, system, info, results, output_csv_file, args.num_runs)
                     except BenchmarkError:
                         pass  # nothing to be done
                     else:
@@ -513,6 +512,8 @@ def run_benchmarks(args: argparse.Namespace) -> None:
                         help='Specify file to write measurement in CSV format')
     parser.add_argument('--args', dest='binargs', metavar='ARGS', default=None, action='store',
                         help='provide additional arguments to pass through to the binary')
+    parser.add_argument('-n', '--num-runs', dest='num_runs', metavar='RUNS', default=5, action='store', type=int,
+                        help='specify the number of runs per system, configuration, and case')
     parser.add_argument('-v', '--verbose', help='verbose output', dest='verbose', default=False, action='store_true')
     parser.add_argument('--pgsql', dest='pgsql', default=False, action='store_true',
                         help='create a .pgsql file with instructions to insert measurement results into a PostgreSQL '