Add line plots

post-processing/README.md

@@ -32,12 +32,11 @@ pip install -e .[post-processing]
 #### Command line
 
 ```sh
-python post_processing.py log_path config_path [-p plot_type]
+python post_processing.py log_path config_path
 ```
 
 - `log_path` - Path to a perflog file, or a directory containing perflog files.
 - `config_path` - Path to a configuration file containing plot details.
-- `plot_type` - (Optional.) Type of plot to be generated. (`Note: only a generic bar chart is currently implemented.`)
 
 Run `post_processing.py -h` for more information (including debugging and file output flags).
 

post-processing/config_handler.py

@@ -19,6 +19,7 @@ def __init__(self, config: dict, template=False):
             config = read_config(config)
 
         # extract config information
+        self.plot_type = config.get("plot_type")
         self.title = config.get("title")
         self.x_axis = config.get("x_axis")
         self.y_axis = config.get("y_axis")
@@ -60,9 +61,10 @@ def from_template(self):
         """
 
         return self(dict({
+            "plot_type": None,
             "title": None,
-            "x_axis": {"value": None, "units": {"custom": None}},
-            "y_axis": {"value": None, "units": {"custom": None}, "scaling": {"custom": None}},
+            "x_axis": {"value": None, "units": {"custom": None}, "range": {"use_default": True, "min": None, "max": None}},
+            "y_axis": {"value": None, "units": {"custom": None}, "scaling": {"custom": None}, "range": {"use_default": True, "min": None, "max": None}},
             "filters": {"and": [], "or": []},
             "series": [],
             "column_types": {},
@@ -191,6 +193,7 @@ def to_dict(self):
         """
 
         return dict({
+            "plot_type": self.plot_type,
             "title": self.title,
             "x_axis": self.x_axis,
             "y_axis": self.y_axis,
@@ -237,6 +240,13 @@ def read_config(config: dict):
             config: dict, plot configuration information.
     """
 
+    # check plot_type information
+    plot_type = config.get("plot_type")
+    if not plot_type:
+        raise KeyError("Missing plot type information.")
+    elif (plot_type != 'generic') and (plot_type != 'line'):
+        raise RuntimeError("Plot type must be one of 'generic' or 'line'.")
+
     # check plot title information
     if not config.get("title"):
         raise KeyError("Missing plot title information.")
@@ -248,6 +258,8 @@ def read_config(config: dict):
         raise KeyError("Missing x-axis value information.")
     if not config.get("x_axis").get("units"):
         raise KeyError("Missing x-axis units information.")
+    if not config.get("x_axis").get("range"):
+        raise KeyError("Missing x-axis range information.")
     if (config.get("x_axis").get("units").get("column") is not None and
         config.get("x_axis").get("units").get("custom") is not None):
         raise RuntimeError(
@@ -260,6 +272,8 @@ def read_config(config: dict):
         raise KeyError("Missing y-axis value information.")
     if not config.get("y_axis").get("units"):
         raise KeyError("Missing y-axis units information.")
+    if not config.get("y_axis").get("range"):
+        raise KeyError("Missing y-axis range information.")
     if (config.get("y_axis").get("units").get("column") is not None and
         config.get("y_axis").get("units").get("custom") is not None):
         raise RuntimeError(
@@ -280,8 +294,12 @@ def read_config(config: dict):
 
     # check optional series information
     if config.get("series"):
-        if len(config.get("series")) == 1:
-            raise RuntimeError("Number of series must be >= 2.")
+        if plot_type == 'generic':
+            if len(config.get("series")) == 1:
+                raise RuntimeError("Number of series must be >= 2 for generic plot.")
+        if plot_type == 'line':
+            if len(config.get("series")) < 1:
+                raise RuntimeError("Number of series must be >= 1 for line plot.")
         if len(set([s[0] for s in config.get("series")])) > 1:
             raise RuntimeError("Currently supporting grouping of series by only one column. \
                                Please use a single column name in your series configuration.")

post-processing/plot_handler.py

@@ -4,12 +4,100 @@
 
 import numpy as np
 import pandas as pd
+from pandas.api.types import is_datetime64_any_dtype as is_datetime
 from bokeh.models import HoverTool, Legend
 from bokeh.models.sources import ColumnDataSource
 from bokeh.palettes import viridis
 from bokeh.plotting import figure, output_file, save
 from bokeh.transform import factor_cmap
 from titlecase import titlecase
+import itertools
+
+def get_axis_min_max(df, axis, column):
+    axis_range = axis["range"]
+    axis_min = axis_range["min"] if axis_range["min"] != 'None' else None
+    axis_max = axis_range["max"] if axis_range["max"] != 'None' else None
+
+    #FIXME: str types and user defined datetime ranges not currently supported
+    # use defaults if type is datetime
+    axis_min_element = np.nanmin(df[column])
+    axis_max_element = np.nanmax(df[column])
+    if (is_datetime(df[column])):
+        datetime_range = axis_max_element - axis_min_element
+        buffer_time = datetime_range*0.2
+        axis_min = axis_min_element - buffer_time
+        axis_max = axis_max_element + buffer_time
+    else:
+        if not (axis_min and axis_max):
+            axis_min = (axis_min_element*0.6 if min(df[column]) >= 0
+                    else math.floor(axis_min_element*1.2))
+            axis_max = (axis_max_element*0.6 if max(df[column]) <= 0
+                    else math.ceil(axis_max_element*1.2))
+
+    return axis_min, axis_max
+
+def plot_line_chart(title, df: pd.DataFrame, x_axis, y_axis, series_filters):
+    """
+        Create a line chart for the supplied data using bokeh.
+
+        Args:
+            title: str, plot title.
+            df: dataframe, data to plot.
+            x_axis: dict, x-axis column and units.
+            y_axis: dict, y-axis column and units.
+            series_filters: list, x-axis groups used to filter graph data.
+    """
+    # get column names and labels for axes
+    x_column, x_label = get_axis_labels(df, x_axis, series_filters)
+    y_column, y_label = get_axis_labels(df, y_axis, series_filters)
+
+    min_x, max_x = get_axis_min_max(df, x_axis, x_column)
+    min_y, max_y = get_axis_min_max(df, y_axis, y_column)
+
+    # create html file to store plot in
+    output_file(filename=os.path.join(
+        Path(__file__).parent, "{0}.html".format(title.replace(" ", "_"))), title=title)
+
+    # create plot
+    plot = figure(x_range=(min_x, max_x), y_range=(min_y, max_y), title=title,
+                  width=800, toolbar_location="above")
+
+    # configure tooltip
+    plot.add_tools(HoverTool(tooltips=[
+                                (y_label, "@{0}".format(y_column)
+                                    + ("{%0.2f}" if pd.api.types.is_float_dtype(df[y_column].dtype)
+                                       else ""))],
+                             formatters={"@{0}".format(y_column): "printf"}))
+
+    # create legend outside plot
+    plot.add_layout(Legend(), "right")
+
+    colours = itertools.cycle(viridis(len(series_filters)))
+
+    for filter in series_filters:
+        filtered_df = None
+        if filter[1] == '==':
+            filtered_df = df[df[filter[0]] == int(filter[2])]
+            plot.line(x=x_column, y=y_column, source=filtered_df, legend_label=' '.join(filter), line_width=2, color=next(colours))
+
+    # add labels
+    plot.xaxis.axis_label = x_label
+    plot.yaxis.axis_label = y_label
+    # adjust font size
+    plot.title.text_font_size = "15pt"
+
+    # flip x-axis if sort is descending
+    if x_axis.get("sort"):
+        if x_axis["sort"] == "descending":
+            end = plot.x_range.end
+            start = plot.x_range.start
+            plot.x_range.start = end
+            plot.x_range.end = start
+
+    # save to file
+    save(plot)
+
+    return plot
 
 
 def plot_generic(title, df: pd.DataFrame, x_axis, y_axis, series_filters, debug=False):

post-processing/post_processing.py

@@ -8,7 +8,7 @@
 import pandas as pd
 from config_handler import ConfigHandler
 from perflog_handler import PerflogHandler
-from plot_handler import plot_generic
+from plot_handler import plot_generic, plot_line_chart
 
 
 class PostProcessing:
@@ -73,9 +73,14 @@ def run_post_processing(self, config: ConfigHandler):
 
         # call a plotting script
         if self.plotting:
-            self.plot = plot_generic(
-                config.title, self.df[self.mask][config.plot_columns],
-                config.x_axis, config.y_axis, config.series_filters, self.debug)
+            if config.plot_type == 'generic':
+                self.plot = plot_generic(
+                    config.title, self.df[self.mask][config.plot_columns],
+                    config.x_axis, config.y_axis, config.series_filters, self.debug)
+            elif config.plot_type == 'line':
+                self.plot = plot_line_chart(
+                    config.title, self.df[self.mask][config.plot_columns],
+                    config.x_axis, config.y_axis, config.series_filters)
 
         # FIXME (#issue #255): maybe save this bit to a file as well for easier viewing
         if self.debug & self.verbose:
@@ -416,10 +421,6 @@ def read_args():
     parser.add_argument("config_path", type=Path,
                         help="path to a configuration file specifying what to plot")
 
-    # optional argument (plot type)
-    parser.add_argument("-p", "--plot_type", type=str, default="generic",
-                        help="type of plot to be generated (default: 'generic')")
-
     # info dump flags
     parser.add_argument("-d", "--debug", action="store_true",
                         help="debug flag for printing additional information")

post-processing/streamlit_post_processing.py

@@ -68,6 +68,14 @@ def update_ui(post: PostProcessing, config: ConfigHandler, e: 'Exception | None'
         # config file uploader
         st.file_uploader("Upload Config", type="yaml", key="uploaded_config", on_change=update_config)
 
+        # set plot type
+        plot_type_options = ['generic', 'line']
+        plot_type_index = plot_type_options.index(config.plot_type) if config.plot_type else None
+        plot_type = st.selectbox("#### Plot type", plot_type_options,
+            key="plot_type", index=plot_type_index)
+        if plot_type != config.plot_type:
+            config.plot_type = plot_type
+
         # set plot title
         title = st.text_input("#### Title", config.title, placeholder="None")
         if title != config.title:
@@ -169,6 +177,16 @@ def axis_select(label: str, axis: dict):
 
     # units select
     units_select(label, axis)
+
+    # axis range
+    use_default_ranges = st.checkbox("Use default ranges", axis.get("range").get("use_default"), key="{0}_axis_range_use_default".format(label))
+    if not use_default_ranges:
+        axis_range_min, axis_range_max = st.columns(2)
+        with axis_range_min:
+            st.number_input("Minimum", key="{0}_axis_range_min".format(label))
+        with axis_range_max:
+            st.number_input("Maximum", key="{0}_axis_range_max".format(label))
+
     # scaling select
     if label == "y":
         st.write("---")
@@ -286,6 +304,9 @@ def update_axes():
     x_column = state.x_axis_column
     x_units_column = state.x_axis_units_column
     x_units_custom = state.x_axis_units_custom
+    x_range_use_default = state.x_axis_range_use_default
+    x_range_min = None if x_range_use_default else state.x_axis_range_min
+    x_range_max = None if x_range_use_default else state.x_axis_range_max
 
     y_column = state.y_axis_column
     y_units_column = state.y_axis_units_column
@@ -294,6 +315,9 @@ def update_axes():
     y_scaling_series = state.y_axis_scaling_series
     y_scaling_x = state.y_axis_scaling_x_value
     y_scaling_custom = state.y_axis_custom_scaling_val
+    y_range_use_default = state.y_axis_range_use_default
+    y_range_min = None if y_range_use_default else state.y_axis_range_min
+    y_range_max = None if y_range_use_default else state.y_axis_range_max
 
     # update columns
     config.x_axis["value"] = x_column
@@ -313,6 +337,13 @@ def update_axes():
         config.y_axis["units"] = {"column": y_units_column}
         config.column_types[y_units_column] = "str"
 
+    config.x_axis["range"]["use_default"] = x_range_use_default
+    config.x_axis["range"]["min"] = x_range_min
+    config.x_axis["range"]["max"] = x_range_max
+    config.y_axis["range"]["use_default"] = y_range_use_default
+    config.y_axis["range"]["min"] = y_range_min
+    config.y_axis["range"]["max"] = y_range_max
+
     # update scaling
     config.y_axis["scaling"] = {"custom": y_scaling_custom if y_scaling_custom else None}
     if not y_scaling_custom and y_scaling_column: