run_wrk_benchmark and plotting scripts

benchmark/wrk_http_bm/plot_wrk_bm_histograms.py

@@ -0,0 +1,157 @@
+"""
+Script: plot_wrk_bm_histograms.py
+
+Description:
+This script parses `wrk` or `wrk2` output files generated by the 'run_wrk_bm.py' script 
+and visualizes key metrics in the form of histograms. The histograms focus on latency percentiles 
+and requests per second (RPS) for individual test runs.
+
+Key Features:
+- Parses raw `wrk` output files to extract latency percentiles (50%, 75%, 90%, 99%) and RPS.
+- Visualizes latency percentiles and RPS metrics as histograms for each test run.
+- Saves the output as PNG files for easy analysis and reporting.
+
+Metrics Visualized:
+1. Latency Percentiles (50%, 75%, 90%, 99%)
+2. Requests Per Second (RPS)
+
+Usage:
+1. Run the 'run_wrk_bm.py' script to generate the `wrk` output files.
+2. Use this script to parse the output and generate histograms.
+
+Example:
+    python plot_wrk_bm_histograms.py -i ./wrk_output.txt -o ./plots
+"""
+
+import re
+import matplotlib.pyplot as plt
+import argparse
+import os
+
+def parse_wrk_output(file_path):
+    with open(file_path, 'r') as file:
+        data = file.read()
+
+    # Regex to extract latency distributions and requests/sec
+    regex = r"Running.*?@\s+(.*?)\n.*?(\d+)\sthreads and (\d+)\sconnections.*?Latency Distribution.*?50%\s+([\d\.]+)(ms|s)\s+75%\s+([\d\.]+)(ms|s)\s+90%\s+([\d\.]+)(ms|s)\s+99%\s+([\d\.]+)(ms|s).*?Requests/sec:\s+([\d\.]+)"
+    matches = re.findall(regex, data, re.DOTALL)
+
+    results = []
+    for match in matches:
+        url = match[0]
+        threads = int(match[1])
+        connections = int(match[2])
+        latencies = {
+            "50%": float(match[3]) * (1000 if match[4] == "s" else 1),
+            "75%": float(match[5]) * (1000 if match[6] == "s" else 1),
+            "90%": float(match[7]) * (1000 if match[8] == "s" else 1),
+            "99%": float(match[9]) * (1000 if match[10] == "s" else 1),
+        }
+        rps = float(match[11])  # Requests per second
+        results.append({
+            "url": url,
+            "threads": threads,
+            "connections": connections,
+            "latencies": latencies,
+            "requests_per_sec": rps,
+        })
+
+    return results
+
+def plot_histograms(results, output_file, enable_show=False):
+    colors = ["blue", "orange", "green", "red", "purple", "brown"]  # Colors for each run
+
+    # Prepare a figure with two subplots
+    fig, axes = plt.subplots(2, 1, figsize=(12, 12))
+
+    # Subplot 1: Latency Percentiles
+    for idx, result in enumerate(results):
+        latencies = result["latencies"]
+        percentiles = list(latencies.keys())
+        values = list(latencies.values())
+
+        bars = axes[0].bar(
+            [f"{percentile} (Run {idx+1})" for percentile in percentiles],
+            values,
+            color=colors[idx % len(colors)],
+            alpha=0.7,
+            label=f"{result['url']} (Threads: {result['threads']}, Conns: {result['connections']})",
+        )
+
+        # add text labels on the bars
+        for bar in bars:
+            height = bar.get_height()
+            axes[0].text(
+                bar.get_x() + bar.get_width() / 2,
+                height,
+                f'{height:.2f}',
+                ha='center',
+                va='bottom',
+                fontsize=10
+            )
+
+    axes[0].set_title("Latency Percentiles Across Runs", fontsize=16)
+    axes[0].set_ylabel("Latency (ms)", fontsize=12)
+    axes[0].tick_params(axis="x", rotation=45, labelsize=10)
+    axes[0].legend(loc="upper left", fontsize=10)
+    axes[0].grid(axis="y", linestyle="--", alpha=0.7)
+
+    # Subplot 2: Requests Per Second (RPS)
+    for idx, result in enumerate(results):
+        rps = result["requests_per_sec"]
+
+        bars = axes[1].bar(
+            [f"Run {idx+1}"],
+            [rps],
+            color=colors[idx % len(colors)],
+            alpha=0.7,
+            label=f"{result['url']} (Threads: {result['threads']}, Conns: {result['connections']})",
+        )
+
+        # Add text labels on the bars
+        for bar in bars:
+            height = bar.get_height()
+            axes[1].text(
+                bar.get_x() + bar.get_width() / 2,
+                height,
+                f'{height:.2f}',
+                ha='center',
+                va='bottom',
+                fontsize=10
+            )
+
+    axes[1].set_title("Requests Per Second (RPS) Across Runs", fontsize=16)
+    axes[1].set_ylabel("Requests/sec", fontsize=12)
+    axes[1].tick_params(axis="x", rotation=45, labelsize=10)
+    axes[1].legend(loc="upper left", fontsize=10)
+    axes[1].grid(axis="y", linestyle="--", alpha=0.7)
+
+    # Adjust layout and save the figure
+    plt.tight_layout()
+    plt.savefig(output_file)
+
+    if enable_show:
+        plt.show()
+
+    plt.close()
+    print(f"Saved histogram to {output_file}")
+
+if __name__ == "__main__":
+    # Argument parsing
+    parser = argparse.ArgumentParser(description="Generate histograms for wrk output.")
+    parser.add_argument("-i", "--input_file", type=str,  help="Path to the wrk output file.")
+    parser.add_argument("-o", "--output_folder", type=str, help="Path to the folder to save the plots.")
+    args = parser.parse_args()
+
+    # Ensure the output folder exists
+    os.makedirs(args.output_folder, exist_ok=True)
+
+    # Parse wrk output and generate plots
+    results = parse_wrk_output(args.input_file)
+
+    # Generate output file name based on input file name
+    input_file_name = os.path.splitext(os.path.basename(args.input_file))[0]
+    output_file_name = input_file_name.replace("_results", "_histogram") + ".png"
+    output_file_path = os.path.join(args.output_folder, output_file_name)
+
+    plot_histograms(results, output_file_path)

benchmark/wrk_http_bm/plot_wrk_bm_tests_comparison.py

@@ -0,0 +1,155 @@
+"""
+Script: plot_wrk_bm_tests_comparison.py
+
+Description:
+This script parses CSV files generated by the 'run_wrk_bm.py' tool to create comparative plots 
+of latency percentiles and requests per second (RPS) across multiple benchmark test cases.
+The script reads the CSV files, processes data grouped by unique combinations of thread/connection counts, 
+and generates plots for each combination.
+
+Key Features:
+- Parses CSV files to extract latency and RPS data.
+- Groups results by thread/connection combinations.
+- Generates comparative plots of latency percentiles and RPS across test cases.
+- Saves the output as PNG files for easy visualization.
+
+Usage:
+1. Ensure the CSV files (e.g., bm_results_<test_case>.csv) are generated by the 'run_wrk_bm.py' tool.
+2. Run this script, specifying the folder containing the CSV files and an output folder for the plots.
+
+Example:
+    python plot_wrk_bm_tests_comparison.py -i ./csv_files -o ./plots
+"""
+import os
+import argparse
+import pandas as pd
+import matplotlib.pyplot as plt
+
+def parse_csv_file(file_path):
+    # Read the CSV file into a pandas DataFrame
+    df = pd.read_csv(file_path, header=0)
+
+    # Rename columns to ensure consistent format
+    df.rename(columns={
+        "Latency (ms)": "latency",
+        "Requests/sec": "requests_per_sec",
+        "Threads": "threads",
+        "Connections": "connections",
+        "API": "url"
+    }, inplace=True)
+
+    # Extract the endpoint postfix from the URL
+    df["url"] = df["url"].apply(lambda x: "/".join(x.split("/", 3)[-1:]))
+    return df
+
+def strip_prefix(url, prefixes):
+    for prefix in prefixes:
+        if prefix in url:
+            return url.split(prefix, 1)[-1]  # Remove the prefix and keep everything after it
+    return url  # Return the original URL if no prefix matches
+
+# generates plots per connection_threads
+def plot_per_connection_threads(dataframes, output_folder):
+
+    # Create a combined DataFrame with all results
+    combined_df = pd.concat(dataframes, keys=[df.name for df in dataframes], names=["test_case"])
+
+    # Ensure column names are consistent
+    if "latency" not in combined_df.columns or "requests_per_sec" not in combined_df.columns:
+        raise ValueError("Required columns (latency, requests_per_sec) are missing in the input data.")
+
+    # Transform `url` column to strip prefixes globally
+    endpoint_skip = ["ufmRest/"]
+    combined_df["url"] = combined_df["url"].apply(lambda url: strip_prefix(url, endpoint_skip))
+
+    # Create unique identifiers for connection_threads
+    combined_df["connection_threads"] = combined_df.apply(lambda x: f"{x['connections']}_{x['threads']}", axis=1)
+
+    # Get unique connection_threads combinations
+    unique_combinations = combined_df["connection_threads"].unique()
+
+    # Generate plots for each connection_threads
+    for combination in unique_combinations:
+
+        subset = combined_df[combined_df["connection_threads"] == combination]
+        endpoint_skip = ["ufmRest/"]
+        endpoints = subset["url"].unique()
+
+        fig, axes = plt.subplots(2, 1, figsize=(14, 12))
+
+        # Subplot 1: Latency Comparison
+        for test_case, group in subset.groupby("test_case"):
+            group["url"] = pd.Categorical(group["url"], categories=endpoints, ordered=True)
+            avg_latency = group.groupby("url", observed=False)["latency"].mean().reindex(endpoints)
+
+            axes[0].plot(
+                avg_latency.index,
+                avg_latency,
+                marker="o",
+                label=f"{test_case} (Avg Latency)"
+            )
+
+        axes[0].set_title(f"Average Latency Comparison (Connections: {combination})", fontsize=16)
+        axes[0].set_ylabel("Latency (ms)", fontsize=12)
+        axes[0].set_xticks(range(len(avg_latency.index)))
+        axes[0].set_xticklabels(avg_latency.index, rotation=45, ha="right")
+
+        axes[0].legend(fontsize=10)
+        axes[0].grid(True)
+
+        # Subplot 2: Requests Per Second Comparison
+        for test_case, group in subset.groupby("test_case"):
+            group["url"] = pd.Categorical(group["url"], categories=endpoints, ordered=True)
+            reqs_per_sec = group.groupby("url", observed=False)["requests_per_sec"].mean().reindex(endpoints)
+
+            axes[1].plot(
+                reqs_per_sec.index,
+                reqs_per_sec,
+                marker="o",
+                label=f"{test_case} (Requests per Second)"
+            )
+
+        axes[1].set_title(f"Requests Per Second Comparison (Connections: {combination})", fontsize=16)
+        axes[1].set_ylabel("Requests per Second", fontsize=12)
+        axes[1].set_xlabel("Endpoints", fontsize=12)
+        axes[1].set_xticks(range(len(reqs_per_sec.index)))
+        axes[1].set_xticklabels(reqs_per_sec.index, rotation=45, ha="right")
+
+        axes[1].legend(fontsize=10)
+        axes[1].grid(True)
+
+        # Save the plot
+        output_file = os.path.join(output_folder, f"bm_comparison_tc_{combination}.png")
+        plt.tight_layout()
+        plt.savefig(output_file)
+        plt.close()
+
+        print(f"Plot (v1.0) saved for connection_threads {combination}: {output_file}")
+
+if __name__ == "__main__":
+
+    # parse arguments
+    parser = argparse.ArgumentParser(description="Compare benchmark results from CSV files and generate plots.")
+    parser.add_argument("-i", "--input_folder", type=str, required=True, help="Path to the folder containing CSV files.")
+    parser.add_argument("-o", "--output_folder", type=str, required=True, help="Path to save the combined output PNG file.")
+    args = parser.parse_args()
+
+    # Read 'CSV files in the input folder ends with '_bm.csv'
+    input_files = [f for f in os.listdir(args.input_folder) if f.endswith("_bm.csv")]
+
+    if not input_files:
+        print("No matching files found in the input folder.")
+        exit(1)
+
+    dataframes = []
+    for file_name in input_files:
+        test_case = os.path.splitext(file_name)[0].replace("_bm", "")
+        file_path = os.path.join(args.input_folder, file_name)
+        df = parse_csv_file(file_path)
+        df.name = test_case  # Assign the test case name to the DataFrame
+        dataframes.append(df)
+
+    os.makedirs(args.output_folder, exist_ok=True)
+
+    # Generate plots per connection_threads
+    plot_per_connection_threads(dataframes, args.output_folder)    

benchmark/wrk_http_bm/readme

@@ -0,0 +1,63 @@
+# run_wrk_bm.py
+run_wrk_bm.py is a Python wrapper for the wrk/wrk2 tools, designed to simplify their usage and provide additional functionality.
+
+# Features
+Simplified Wrk/Wrk2 Integration: Wraps the functionality of wrk and wrk2 for easy execution and configuration.
+Test Configuration Management: Parses and manages test settings from a configuration file.
+Result Visualization: Supports plotting test results for better insights and analysis.
+Configuration Example
+For an example configuration file, refer to:
+UFM/gvvm/authentication_server/benchmark
+
+# run with -h for args list
+python run_wrk_bm.py -h
+
+# Prerequisites:
+pandas, matplotlib (pip install)
+wrk, wrk2 (@see Wrk/Wrk2 Installation guide at the end of this document)
+
+# Wrk/wrk2 Overview
+-------------------
+wrk and wrk2 are high-performance HTTP benchmarking tools designed to test the throughput and latency of web servers or APIs.
+
+# wrk
+A modern HTTP benchmarking tool that generates significant load.
+Features:
+Uses multiple threads and connections for concurrency.
+Provides latency distribution statistics.
+Flexible scripting support in Lua to customize requests.
+Typically used for load testing APIs, web servers, or any HTTP endpoint.
+In wrk, there is no explicit control over the request rate.
+
+# wrk2
+An enhanced version of wrk with a focus on constant request rate testing.
+Differences from wrk:
+Adds rate-limiting (-R flag) to send requests at a constant rate (e.g., 1000 requests/second).
+Ideal for latency benchmarking under steady-state conditions.
+Also supports Lua scripting and provides latency distribution output.
+
+It is built to send requests at a constant rate to simulate real-world scenarios where systems operate at a steady load
+rather than trying to overwhelm them.
+
+Using wrk or wrk2 depends on what you're trying to achieve:
+Use Case Examples:
+wrk: For generating high, unrestricted load to test a server's maximum capacity.
+wrk2: For precisely controlled testing to measure latency at a fixed request rate.
+
+Use wrk for Load Testing:
+If your goal is to find the system's limits: 
+to push the system and identify maximum throughput and bottlenecks.
+
+Use wrk2 for Constant Throughput Testing
+If your goal is to evaluate system behavior under real-world usage scenarios:
+to control the request rate and measure latency and stability under defined conditions.
+
+# Wrk/Wrk2 Installation:
+
+git clone https://github.com/wg/wrk.git
+cd wrk
+make
+
+git clone https://github.com/giltene/wrk2.git
+cd wrk2
+make