Add line plots

post-processing/README.md

@@ -32,12 +32,11 @@ pip install -e .[post-processing]
 #### Command line
 
 ```sh
-python post_processing.py log_path config_path [-p plot_type]
+python post_processing.py log_path config_path
 ```
 
 - `log_path` - Path to a perflog file, or a directory containing perflog files.
 - `config_path` - Path to a configuration file containing plot details.
-- `plot_type` - (Optional.) Type of plot to be generated. (`Note: only a generic bar chart is currently implemented.`)
 
 Run `post_processing.py -h` for more information (including debugging and file output flags).
 

post-processing/config_handler.py

@@ -19,6 +19,7 @@ def __init__(self, config: dict, template=False):
             config = read_config(config)
 
         # extract config information
+        self.plot_type = config.get("plot_type")
         self.title = config.get("title")
         self.x_axis = config.get("x_axis")
         self.y_axis = config.get("y_axis")
@@ -60,9 +61,13 @@ def from_template(self):
         """
 
         return self(dict({
+            "plot_type": None,
             "title": None,
-            "x_axis": {"value": None, "units": {"custom": None}},
-            "y_axis": {"value": None, "units": {"custom": None}, "scaling": {"custom": None}},
+            "x_axis": {"value": None, "units": {"custom": None},
+                       "range": {"min": None, "max": None}},
+            "y_axis": {"value": None, "units": {"custom": None},
+                       "scaling": {"custom": None},
+                       "range": {"min": None, "max": None}},
             "filters": {"and": [], "or": []},
             "series": [],
             "column_types": {},
@@ -191,6 +196,7 @@ def to_dict(self):
         """
 
         return dict({
+            "plot_type": self.plot_type,
             "title": self.title,
             "x_axis": self.x_axis,
             "y_axis": self.y_axis,
@@ -237,6 +243,13 @@ def read_config(config: dict):
             config: dict, plot configuration information.
     """
 
+    # check plot_type information
+    plot_type = config.get("plot_type")
+    if not plot_type:
+        raise KeyError("Missing plot type information.")
+    elif (plot_type != 'generic') and (plot_type != 'line'):
+        raise RuntimeError("Plot type must be one of 'generic' or 'line'.")
+
     # check plot title information
     if not config.get("title"):
         raise KeyError("Missing plot title information.")
@@ -248,6 +261,8 @@ def read_config(config: dict):
         raise KeyError("Missing x-axis value information.")
     if not config.get("x_axis").get("units"):
         raise KeyError("Missing x-axis units information.")
+    if not config.get("x_axis").get("range"):
+        raise KeyError("Missing x-axis range information.")
     if (config.get("x_axis").get("units").get("column") is not None and
         config.get("x_axis").get("units").get("custom") is not None):
         raise RuntimeError(
@@ -260,6 +275,8 @@ def read_config(config: dict):
         raise KeyError("Missing y-axis value information.")
     if not config.get("y_axis").get("units"):
         raise KeyError("Missing y-axis units information.")
+    if not config.get("y_axis").get("range"):
+        raise KeyError("Missing y-axis range information.")
     if (config.get("y_axis").get("units").get("column") is not None and
         config.get("y_axis").get("units").get("custom") is not None):
         raise RuntimeError(
@@ -280,8 +297,12 @@ def read_config(config: dict):
 
     # check optional series information
     if config.get("series"):
-        if len(config.get("series")) == 1:
-            raise RuntimeError("Number of series must be >= 2.")
+        if plot_type == 'generic':
+            if len(config.get("series")) == 1:
+                raise RuntimeError("Number of series must be >= 2 for generic plot.")
+        if plot_type == 'line':
+            if len(config.get("series")) < 1:
+                raise RuntimeError("Number of series must be >= 1 for line plot.")
         if len(set([s[0] for s in config.get("series")])) > 1:
             raise RuntimeError("Currently supporting grouping of series by only one column. \
                                Please use a single column name in your series configuration.")

post-processing/plot_handler.py

@@ -1,3 +1,4 @@
+import itertools
 import math
 import os
 from pathlib import Path
@@ -9,6 +10,7 @@
 from bokeh.palettes import viridis
 from bokeh.plotting import figure, output_file, save
 from bokeh.transform import factor_cmap
+from pandas.api.types import is_datetime64_any_dtype as is_datetime
 from titlecase import titlecase
 
 
@@ -65,10 +67,9 @@ def plot_generic(title, df: pd.DataFrame, x_axis, y_axis, series_filters, debug=
     plot = figure(x_range=grouped_df, y_range=(min_y, max_y), title=title,
                   width=800, toolbar_location="above")
     # configure tooltip
-    plot.add_tools(HoverTool(tooltips=[
-                                (y_label, "@{0}_mean".format(y_column)
-                                    + ("{%0.2f}" if pd.api.types.is_float_dtype(df[y_column].dtype)
-                                       else ""))],
+    plot.add_tools(HoverTool(tooltips=[(y_label, "@{0}_mean".format(y_column)
+                                        + ("{%0.2f}" if pd.api.types.is_float_dtype(df[y_column].dtype)
+                                           else ""))],
                              formatters={"@{0}_mean".format(y_column): "printf"}))
 
     # sort x-axis values in descending order (otherwise default sort is ascending)
@@ -179,3 +180,103 @@ def get_axis_labels(df: pd.DataFrame, axis, series_filters):
                                " ({0})".format(units) if units else "")
 
     return col_name, label
+
+
+def plot_line_chart(title, df: pd.DataFrame, x_axis, y_axis, series_filters):
+    """
+        Create a line chart for the supplied data using bokeh.
+
+        Args:
+            title: str, plot title.
+            df: dataframe, data to plot.
+            x_axis: dict, x-axis column and units.
+            y_axis: dict, y-axis column and units.
+            series_filters: list, x-axis groups used to filter graph data.
+    """
+
+    # get column names and labels for axes
+    x_column, x_label = get_axis_labels(df, x_axis, series_filters)
+    y_column, y_label = get_axis_labels(df, y_axis, series_filters)
+
+    # adjust axis ranges
+    min_x, max_x = get_axis_min_max(df, x_axis)
+    min_y, max_y = get_axis_min_max(df, y_axis)
+
+    # create html file to store plot in
+    output_file(filename=os.path.join(
+        Path(__file__).parent, "{0}.html".format(title.replace(" ", "_"))), title=title)
+
+    # create plot
+    plot = figure(x_range=(min_x, max_x), y_range=(min_y, max_y), title=title,
+                  width=800, toolbar_location="above")
+
+    # configure tooltip
+    plot.add_tools(HoverTool(tooltips=[(y_label, "@{0}".format(y_column)
+                                        + ("{%0.2f}" if pd.api.types.is_float_dtype(df[y_column].dtype)
+                                           else ""))],
+                             formatters={"@{0}".format(y_column): "printf"}))
+
+    # create legend outside plot
+    plot.add_layout(Legend(), "right")
+    colours = itertools.cycle(viridis(len(series_filters)))
+
+    for filter in series_filters:
+        filtered_df = None
+        if filter[1] == '==':
+            filtered_df = df[df[filter[0]] == int(filter[2])]
+            plot.line(x=x_column, y=y_column, source=filtered_df, legend_label=' '.join(filter),
+                      line_width=2, color=next(colours))
+
+    # add labels
+    plot.xaxis.axis_label = x_label
+    plot.yaxis.axis_label = y_label
+    # adjust font size
+    plot.title.text_font_size = "15pt"
+
+    # flip x-axis if sort is descending
+    if x_axis.get("sort"):
+        if x_axis["sort"] == "descending":
+            end = plot.x_range.end
+            start = plot.x_range.start
+            plot.x_range.start = end
+            plot.x_range.end = start
+
+    # save to file
+    save(plot)
+
+    return plot
+
+
+def get_axis_min_max(df, axis):
+    """
+        Return the minimum and maximum numeric values for a given axis.
+
+        Args:
+            df: dataframe, data to plot.
+            axis: dict, axis column, units, and values to scale by.
+    """
+
+    column = axis["value"]
+    axis_range = axis["range"]
+    axis_min = axis_range["min"] if axis_range["min"] else 0.0
+    axis_max = axis_range["max"] if axis_range["max"] else 0.0
+
+    # FIXME: str types and user defined datetime ranges not currently supported
+    if (column):
+        axis_min_element = np.nanmin(df[column])
+        axis_max_element = np.nanmax(df[column])
+
+        # use defaults if type is datetime
+        if (is_datetime(df[column])):
+            datetime_range = axis_max_element - axis_min_element
+            buffer_time = datetime_range*0.2
+            axis_min = axis_min_element - buffer_time
+            axis_max = axis_max_element + buffer_time
+
+        elif axis_min is None or axis_max is None or axis_min == axis_max:
+            axis_min = (axis_min_element*0.6 if min(df[column]) >= 0
+                        else math.floor(axis_min_element*1.2))
+            axis_max = (axis_max_element*0.6 if max(df[column]) <= 0
+                        else math.ceil(axis_max_element*1.2))
+
+    return axis_min, axis_max

post-processing/post_processing.py

@@ -8,7 +8,7 @@
 import pandas as pd
 from config_handler import ConfigHandler
 from perflog_handler import PerflogHandler
-from plot_handler import plot_generic
+from plot_handler import plot_generic, plot_line_chart
 
 
 class PostProcessing:
@@ -73,9 +73,14 @@ def run_post_processing(self, config: ConfigHandler):
 
         # call a plotting script
         if self.plotting:
-            self.plot = plot_generic(
-                config.title, self.df[self.mask][config.plot_columns],
-                config.x_axis, config.y_axis, config.series_filters, self.debug)
+            if config.plot_type == 'generic':
+                self.plot = plot_generic(
+                    config.title, self.df[self.mask][config.plot_columns],
+                    config.x_axis, config.y_axis, config.series_filters, self.debug)
+            elif config.plot_type == 'line':
+                self.plot = plot_line_chart(
+                    config.title, self.df[self.mask][config.plot_columns],
+                    config.x_axis, config.y_axis, config.series_filters)
 
         # FIXME (#issue #255): maybe save this bit to a file as well for easier viewing
         if self.debug & self.verbose:
@@ -416,10 +421,6 @@ def read_args():
     parser.add_argument("config_path", type=Path,
                         help="path to a configuration file specifying what to plot")
 
-    # optional argument (plot type)
-    parser.add_argument("-p", "--plot_type", type=str, default="generic",
-                        help="type of plot to be generated (default: 'generic')")
-
     # info dump flags
     parser.add_argument("-d", "--debug", action="store_true",
                         help="debug flag for printing additional information")

post-processing/streamlit_post_processing.py

@@ -5,6 +5,7 @@
 import streamlit as st
 from config_handler import ConfigHandler, load_config, read_config
 from post_processing import PostProcessing
+from plot_handler import get_axis_min_max
 
 # drop-down lists
 operators = ["==", "!=", "<", ">", "<=", ">="]
@@ -68,6 +69,14 @@ def update_ui(post: PostProcessing, config: ConfigHandler, e: 'Exception | None'
         # config file uploader
         st.file_uploader("Upload Config", type="yaml", key="uploaded_config", on_change=update_config)
 
+        # set plot type
+        plot_type_options = ['generic', 'line']
+        plot_type_index = plot_type_options.index(config.plot_type) if config.plot_type else 0
+        plot_type = st.selectbox("#### Plot type", plot_type_options,
+                                 key="plot_type", index=plot_type_index)
+        if plot_type != config.plot_type:
+            config.plot_type = plot_type
+
         # set plot title
         title = st.text_input("#### Title", config.title, placeholder="None")
         if title != config.title:
@@ -128,18 +137,18 @@ def axis_options():
 
     with st.container(border=True):
         # x-axis select
-        axis_select("x", config.x_axis)
+        axis_select("x", config.x_axis, config.plot_type)
         sort = st.checkbox("sort descending", True if config.x_axis.get("sort") == "descending" else False)
     with st.container(border=True):
         # y-axis select
-        axis_select("y", config.y_axis)
+        axis_select("y", config.y_axis, config.plot_type)
 
     # apply changes
     update_axes()
     config.x_axis["sort"] = "descending" if sort else "ascending"
 
 
-def axis_select(label: str, axis: dict):
+def axis_select(label: str, axis: dict, plot_type: str):
     """
         Allow the user to select axis column and type for post-processing.
 
@@ -169,6 +178,19 @@ def axis_select(label: str, axis: dict):
 
     # units select
     units_select(label, axis)
+
+    # FIXME: add ability to use a custom value for only one of min or max
+    range = get_axis_min_max(df, axis)
+    axis_range_min, axis_range_max = st.columns(2)
+    with axis_range_min:
+        st.number_input("{0}-axis minimum".format(label),
+                        value=range[0],
+                        key="{0}_axis_range_min".format(label))
+    with axis_range_max:
+        st.number_input("{0}-axis maximum".format(label),
+                        value=range[1],
+                        key="{0}_axis_range_max".format(label))
+
     # scaling select
     if label == "y":
         st.write("---")
@@ -286,6 +308,8 @@ def update_axes():
     x_column = state.x_axis_column
     x_units_column = state.x_axis_units_column
     x_units_custom = state.x_axis_units_custom
+    x_range_min = state.x_axis_range_min
+    x_range_max = state.x_axis_range_max
 
     y_column = state.y_axis_column
     y_units_column = state.y_axis_units_column
@@ -294,6 +318,8 @@ def update_axes():
     y_scaling_series = state.y_axis_scaling_series
     y_scaling_x = state.y_axis_scaling_x_value
     y_scaling_custom = state.y_axis_custom_scaling_val
+    y_range_min = state.y_axis_range_min
+    y_range_max = state.y_axis_range_max
 
     # update columns
     config.x_axis["value"] = x_column
@@ -313,6 +339,11 @@ def update_axes():
         config.y_axis["units"] = {"column": y_units_column}
         config.column_types[y_units_column] = "str"
 
+    config.x_axis["range"]["min"] = x_range_min
+    config.x_axis["range"]["max"] = x_range_max
+    config.y_axis["range"]["min"] = y_range_min
+    config.y_axis["range"]["max"] = y_range_max
+
     # update scaling
     config.y_axis["scaling"] = {"custom": y_scaling_custom if y_scaling_custom else None}
     if not y_scaling_custom and y_scaling_column: