From e6f72bc13ff773e79880262b92cbec453bb9431d Mon Sep 17 00:00:00 2001 From: Connor Aird Date: Wed, 22 May 2024 18:04:05 +0100 Subject: [PATCH 1/8] Add option to plot line charts --- post-processing/config_handler.py | 18 ++++- post-processing/plot_handler.py | 69 ++++++++++++++++++++ post-processing/post_processing.py | 13 ++-- post-processing/streamlit_post_processing.py | 6 ++ 4 files changed, 100 insertions(+), 6 deletions(-) diff --git a/post-processing/config_handler.py b/post-processing/config_handler.py index 8a33bc33..90117554 100644 --- a/post-processing/config_handler.py +++ b/post-processing/config_handler.py @@ -19,6 +19,7 @@ def __init__(self, config: dict, template=False): config = read_config(config) # extract config information + self.plot_type = config.get("plot_type") self.title = config.get("title") self.x_axis = config.get("x_axis") self.y_axis = config.get("y_axis") @@ -60,6 +61,7 @@ def from_template(self): """ return self(dict({ + "plot_type": "generic", "title": None, "x_axis": {"value": None, "units": {"custom": None}}, "y_axis": {"value": None, "units": {"custom": None}, "scaling": {"custom": None}}, @@ -191,6 +193,7 @@ def to_dict(self): """ return dict({ + "plot_type": self.plot_type, "title": self.title, "x_axis": self.x_axis, "y_axis": self.y_axis, @@ -237,6 +240,13 @@ def read_config(config: dict): config: dict, plot configuration information. """ + # check plot plot_type information + plot_type = not config.get("plot_type") + if not plot_type: + raise KeyError("Missing plot type information.") + elif (plot_type != 'generic') | (plot_type != 'line'): + raise RuntimeError("plot_type must be one of: 'generic', 'line'") + # check plot title information if not config.get("title"): raise KeyError("Missing plot title information.") @@ -280,8 +290,12 @@ def read_config(config: dict): # check optional series information if config.get("series"): - if len(config.get("series")) == 1: - raise RuntimeError("Number of series must be >= 2.") + if plot_type == 'generic': + if len(config.get("series")) == 1: + raise RuntimeError("Number of series must be >= 2 for generic plot.") + if plot_type == 'line': + if len(config.get("series")) < 1: + raise RuntimeError("Number of series must be >= 1 for line plot.") if len(set([s[0] for s in config.get("series")])) > 1: raise RuntimeError("Currently supporting grouping of series by only one column. \ Please use a single column name in your series configuration.") diff --git a/post-processing/plot_handler.py b/post-processing/plot_handler.py index 62cd650b..75ebf246 100644 --- a/post-processing/plot_handler.py +++ b/post-processing/plot_handler.py @@ -11,6 +11,75 @@ from bokeh.transform import factor_cmap from titlecase import titlecase +def plot_line_chart(title, df: pd.DataFrame, x_axis, y_axis, series_filters, debug=False): + """ + Create a bar chart for the supplied data using bokeh. + + Args: + title: str, plot title. + df: dataframe, data to plot. + x_axis: dict, x-axis column and units. + y_axis: dict, y-axis column and units. + series_filters: list, x-axis groups used to filter graph data. + """ + + # get column names and labels for axes + x_column, x_label = get_axis_labels(df, x_axis, series_filters) + y_column, y_label = get_axis_labels(df, y_axis, series_filters) + + # adjust axis ranges + min_y = (0 if min(df[y_column]) >= 0 + else math.floor(np.nanmin(df[y_column])*1.2)) + max_y = (0 if max(df[y_column]) <= 0 + else math.ceil(np.nanmax(df[y_column])*1.2)) + min_x = (0 if min(df[x_column]) >= 0 + else math.floor(np.nanmin(df[x_column])*1.2)) + max_x = (0 if max(df[x_column]) <= 0 + else math.ceil(np.nanmax(df[x_column])*1.2)) + + # create html file to store plot in + output_file(filename=os.path.join( + Path(__file__).parent, "{0}.html".format(title.replace(" ", "_"))), title=title) + + # create plot + plot = figure(x_range=(min_x, max_x), y_range=(min_y, max_y), title=title, + width=800, toolbar_location="above") + + # configure tooltip + plot.add_tools(HoverTool(tooltips=[ + (y_label, "@{0}".format(y_column) + + ("{%0.2f}" if pd.api.types.is_float_dtype(df[y_column].dtype) + else ""))], + formatters={"@{0}".format(y_column): "printf"})) + + # create legend outside plot + plot.add_layout(Legend(), "right") + + for filter in series_filters: + filtered_df = None + if filter[1] == '==': + filtered_df = df[df[filter[0]] == int(filter[2])] + plot.line(x=x_column, y=y_column, source=filtered_df, legend_label=' '.join(filter), line_width=2) + + # add labels + plot.xaxis.axis_label = x_label + plot.yaxis.axis_label = y_label + # adjust font size + plot.title.text_font_size = "15pt" + + # flip x-axis if sort is descending + if x_axis.get("sort"): + if x_axis["sort"] == "descending": + end = plot.x_range.end + start = plot.x_range.start + plot.x_range.start = end + plot.x_range.end = start + + # save to file + save(plot) + + return plot + def plot_generic(title, df: pd.DataFrame, x_axis, y_axis, series_filters, debug=False): """ diff --git a/post-processing/post_processing.py b/post-processing/post_processing.py index a5b1e8d7..60564abe 100644 --- a/post-processing/post_processing.py +++ b/post-processing/post_processing.py @@ -8,7 +8,7 @@ import pandas as pd from config_handler import ConfigHandler from perflog_handler import PerflogHandler -from plot_handler import plot_generic +from plot_handler import plot_generic, plot_line_chart class PostProcessing: @@ -73,9 +73,14 @@ def run_post_processing(self, config: ConfigHandler): # call a plotting script if self.plotting: - self.plot = plot_generic( - config.title, self.df[self.mask][config.plot_columns], - config.x_axis, config.y_axis, config.series_filters, self.debug) + if config.plot_type == 'generic': + self.plot = plot_generic( + config.title, self.df[self.mask][config.plot_columns], + config.x_axis, config.y_axis, config.series_filters, self.debug) + elif config.plot_type == 'line': + self.plot = plot_line_chart( + config.title, self.df[self.mask][config.plot_columns], + config.x_axis, config.y_axis, config.series_filters, self.debug) # FIXME (#issue #255): maybe save this bit to a file as well for easier viewing if self.debug & self.verbose: diff --git a/post-processing/streamlit_post_processing.py b/post-processing/streamlit_post_processing.py index c6426a02..e1a49030 100644 --- a/post-processing/streamlit_post_processing.py +++ b/post-processing/streamlit_post_processing.py @@ -68,6 +68,12 @@ def update_ui(post: PostProcessing, config: ConfigHandler, e: 'Exception | None' # config file uploader st.file_uploader("Upload Config", type="yaml", key="uploaded_config", on_change=update_config) + # set plot type + plot_type = st.selectbox("#### Plot type", ['generic', 'line'], + key="plot_type") + if plot_type != config.plot_type: + config.plot_type = plot_type + # set plot title title = st.text_input("#### Title", config.title, placeholder="None") if title != config.title: From 94288ef20d7ba0a4ab571efccb53f2406c1c0cd6 Mon Sep 17 00:00:00 2001 From: Connor Aird Date: Thu, 23 May 2024 10:04:37 +0100 Subject: [PATCH 2/8] Fix config check bug and add colours to lines --- post-processing/config_handler.py | 6 +++--- post-processing/plot_handler.py | 10 ++++++---- post-processing/post_processing.py | 2 +- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/post-processing/config_handler.py b/post-processing/config_handler.py index 90117554..9c9c4b46 100644 --- a/post-processing/config_handler.py +++ b/post-processing/config_handler.py @@ -240,11 +240,11 @@ def read_config(config: dict): config: dict, plot configuration information. """ - # check plot plot_type information - plot_type = not config.get("plot_type") + # check plot_type information + plot_type = config.get("plot_type") if not plot_type: raise KeyError("Missing plot type information.") - elif (plot_type != 'generic') | (plot_type != 'line'): + elif (plot_type != 'generic') and (plot_type != 'line'): raise RuntimeError("plot_type must be one of: 'generic', 'line'") # check plot title information diff --git a/post-processing/plot_handler.py b/post-processing/plot_handler.py index 75ebf246..5eed0f68 100644 --- a/post-processing/plot_handler.py +++ b/post-processing/plot_handler.py @@ -10,10 +10,11 @@ from bokeh.plotting import figure, output_file, save from bokeh.transform import factor_cmap from titlecase import titlecase +import itertools -def plot_line_chart(title, df: pd.DataFrame, x_axis, y_axis, series_filters, debug=False): +def plot_line_chart(title, df: pd.DataFrame, x_axis, y_axis, series_filters): """ - Create a bar chart for the supplied data using bokeh. + Create a line chart for the supplied data using bokeh. Args: title: str, plot title. @@ -22,7 +23,6 @@ def plot_line_chart(title, df: pd.DataFrame, x_axis, y_axis, series_filters, deb y_axis: dict, y-axis column and units. series_filters: list, x-axis groups used to filter graph data. """ - # get column names and labels for axes x_column, x_label = get_axis_labels(df, x_axis, series_filters) y_column, y_label = get_axis_labels(df, y_axis, series_filters) @@ -55,11 +55,13 @@ def plot_line_chart(title, df: pd.DataFrame, x_axis, y_axis, series_filters, deb # create legend outside plot plot.add_layout(Legend(), "right") + colours = itertools.cycle(viridis(len(series_filters))) + for filter in series_filters: filtered_df = None if filter[1] == '==': filtered_df = df[df[filter[0]] == int(filter[2])] - plot.line(x=x_column, y=y_column, source=filtered_df, legend_label=' '.join(filter), line_width=2) + plot.line(x=x_column, y=y_column, source=filtered_df, legend_label=' '.join(filter), line_width=2, color=next(colours)) # add labels plot.xaxis.axis_label = x_label diff --git a/post-processing/post_processing.py b/post-processing/post_processing.py index 60564abe..80efcae9 100644 --- a/post-processing/post_processing.py +++ b/post-processing/post_processing.py @@ -80,7 +80,7 @@ def run_post_processing(self, config: ConfigHandler): elif config.plot_type == 'line': self.plot = plot_line_chart( config.title, self.df[self.mask][config.plot_columns], - config.x_axis, config.y_axis, config.series_filters, self.debug) + config.x_axis, config.y_axis, config.series_filters) # FIXME (#issue #255): maybe save this bit to a file as well for easier viewing if self.debug & self.verbose: From 76c06b99e54d8f2ca79730d192a69a7f4d660c35 Mon Sep 17 00:00:00 2001 From: Connor Aird Date: Thu, 23 May 2024 10:28:33 +0100 Subject: [PATCH 3/8] Removing plot_type CLI option and fixing default generic bug --- post-processing/README.md | 3 +-- post-processing/config_handler.py | 2 +- post-processing/post_processing.py | 4 ---- post-processing/streamlit_post_processing.py | 6 ++++-- 4 files changed, 6 insertions(+), 9 deletions(-) diff --git a/post-processing/README.md b/post-processing/README.md index 73a8dd0a..bb2d7ca4 100644 --- a/post-processing/README.md +++ b/post-processing/README.md @@ -32,12 +32,11 @@ pip install -e .[post-processing] #### Command line ```sh -python post_processing.py log_path config_path [-p plot_type] +python post_processing.py log_path config_path ``` - `log_path` - Path to a perflog file, or a directory containing perflog files. - `config_path` - Path to a configuration file containing plot details. -- `plot_type` - (Optional.) Type of plot to be generated. (`Note: only a generic bar chart is currently implemented.`) Run `post_processing.py -h` for more information (including debugging and file output flags). diff --git a/post-processing/config_handler.py b/post-processing/config_handler.py index 9c9c4b46..a280f7fd 100644 --- a/post-processing/config_handler.py +++ b/post-processing/config_handler.py @@ -61,7 +61,7 @@ def from_template(self): """ return self(dict({ - "plot_type": "generic", + "plot_type": None, "title": None, "x_axis": {"value": None, "units": {"custom": None}}, "y_axis": {"value": None, "units": {"custom": None}, "scaling": {"custom": None}}, diff --git a/post-processing/post_processing.py b/post-processing/post_processing.py index 80efcae9..d3916a47 100644 --- a/post-processing/post_processing.py +++ b/post-processing/post_processing.py @@ -421,10 +421,6 @@ def read_args(): parser.add_argument("config_path", type=Path, help="path to a configuration file specifying what to plot") - # optional argument (plot type) - parser.add_argument("-p", "--plot_type", type=str, default="generic", - help="type of plot to be generated (default: 'generic')") - # info dump flags parser.add_argument("-d", "--debug", action="store_true", help="debug flag for printing additional information") diff --git a/post-processing/streamlit_post_processing.py b/post-processing/streamlit_post_processing.py index e1a49030..247e3104 100644 --- a/post-processing/streamlit_post_processing.py +++ b/post-processing/streamlit_post_processing.py @@ -69,8 +69,10 @@ def update_ui(post: PostProcessing, config: ConfigHandler, e: 'Exception | None' st.file_uploader("Upload Config", type="yaml", key="uploaded_config", on_change=update_config) # set plot type - plot_type = st.selectbox("#### Plot type", ['generic', 'line'], - key="plot_type") + plot_type_options = ['generic', 'line'] + plot_type_index = plot_type_options.index(config.plot_type) if config.plot_type else None + plot_type = st.selectbox("#### Plot type", plot_type_options, + key="plot_type", index=plot_type_index) if plot_type != config.plot_type: config.plot_type = plot_type From 44d7790f4d93a17f1a89d1fc00f7533f75e5e9fb Mon Sep 17 00:00:00 2001 From: Connor Aird Date: Fri, 24 May 2024 13:53:40 +0100 Subject: [PATCH 4/8] Addressing Emily's comments --- post-processing/config_handler.py | 10 +- post-processing/plot_handler.py | 35 +++- post-processing/streamlit_post_processing.py | 23 +++ post-processing/test_post_processing.py | 198 ++++++++++++++----- 4 files changed, 200 insertions(+), 66 deletions(-) diff --git a/post-processing/config_handler.py b/post-processing/config_handler.py index a280f7fd..2a62b1dc 100644 --- a/post-processing/config_handler.py +++ b/post-processing/config_handler.py @@ -63,8 +63,8 @@ def from_template(self): return self(dict({ "plot_type": None, "title": None, - "x_axis": {"value": None, "units": {"custom": None}}, - "y_axis": {"value": None, "units": {"custom": None}, "scaling": {"custom": None}}, + "x_axis": {"value": None, "units": {"custom": None}, "range": {"use_default": True, "min": None, "max": None}}, + "y_axis": {"value": None, "units": {"custom": None}, "scaling": {"custom": None}, "range": {"use_default": True, "min": None, "max": None}}, "filters": {"and": [], "or": []}, "series": [], "column_types": {}, @@ -245,7 +245,7 @@ def read_config(config: dict): if not plot_type: raise KeyError("Missing plot type information.") elif (plot_type != 'generic') and (plot_type != 'line'): - raise RuntimeError("plot_type must be one of: 'generic', 'line'") + raise RuntimeError("Plot type must be one of 'generic' or 'line'.") # check plot title information if not config.get("title"): @@ -258,6 +258,8 @@ def read_config(config: dict): raise KeyError("Missing x-axis value information.") if not config.get("x_axis").get("units"): raise KeyError("Missing x-axis units information.") + if not config.get("x_axis").get("range"): + raise KeyError("Missing x-axis range information.") if (config.get("x_axis").get("units").get("column") is not None and config.get("x_axis").get("units").get("custom") is not None): raise RuntimeError( @@ -270,6 +272,8 @@ def read_config(config: dict): raise KeyError("Missing y-axis value information.") if not config.get("y_axis").get("units"): raise KeyError("Missing y-axis units information.") + if not config.get("y_axis").get("range"): + raise KeyError("Missing y-axis range information.") if (config.get("y_axis").get("units").get("column") is not None and config.get("y_axis").get("units").get("custom") is not None): raise RuntimeError( diff --git a/post-processing/plot_handler.py b/post-processing/plot_handler.py index 5eed0f68..89fc65f0 100644 --- a/post-processing/plot_handler.py +++ b/post-processing/plot_handler.py @@ -4,6 +4,7 @@ import numpy as np import pandas as pd +from pandas.api.types import is_datetime64_any_dtype as is_datetime from bokeh.models import HoverTool, Legend from bokeh.models.sources import ColumnDataSource from bokeh.palettes import viridis @@ -12,6 +13,29 @@ from titlecase import titlecase import itertools +def get_axis_min_max(df, axis, column): + axis_range = axis["range"] + axis_min = axis_range["min"] if axis_range["min"] != 'None' else None + axis_max = axis_range["max"] if axis_range["max"] != 'None' else None + + #FIXME: str types and user defined datetime ranges not currently supported + # use defaults if type is datetime + axis_min_element = np.nanmin(df[column]) + axis_max_element = np.nanmax(df[column]) + if (is_datetime(df[column])): + datetime_range = axis_max_element - axis_min_element + buffer_time = datetime_range*0.2 + axis_min = axis_min_element - buffer_time + axis_max = axis_max_element + buffer_time + else: + if not (axis_min and axis_max): + axis_min = (axis_min_element*0.6 if min(df[column]) >= 0 + else math.floor(axis_min_element*1.2)) + axis_max = (axis_max_element*0.6 if max(df[column]) <= 0 + else math.ceil(axis_max_element*1.2)) + + return axis_min, axis_max + def plot_line_chart(title, df: pd.DataFrame, x_axis, y_axis, series_filters): """ Create a line chart for the supplied data using bokeh. @@ -27,15 +51,8 @@ def plot_line_chart(title, df: pd.DataFrame, x_axis, y_axis, series_filters): x_column, x_label = get_axis_labels(df, x_axis, series_filters) y_column, y_label = get_axis_labels(df, y_axis, series_filters) - # adjust axis ranges - min_y = (0 if min(df[y_column]) >= 0 - else math.floor(np.nanmin(df[y_column])*1.2)) - max_y = (0 if max(df[y_column]) <= 0 - else math.ceil(np.nanmax(df[y_column])*1.2)) - min_x = (0 if min(df[x_column]) >= 0 - else math.floor(np.nanmin(df[x_column])*1.2)) - max_x = (0 if max(df[x_column]) <= 0 - else math.ceil(np.nanmax(df[x_column])*1.2)) + min_x, max_x = get_axis_min_max(df, x_axis, x_column) + min_y, max_y = get_axis_min_max(df, y_axis, y_column) # create html file to store plot in output_file(filename=os.path.join( diff --git a/post-processing/streamlit_post_processing.py b/post-processing/streamlit_post_processing.py index 247e3104..7d9988a3 100644 --- a/post-processing/streamlit_post_processing.py +++ b/post-processing/streamlit_post_processing.py @@ -177,6 +177,16 @@ def axis_select(label: str, axis: dict): # units select units_select(label, axis) + + # axis range + use_default_ranges = st.checkbox("Use default ranges", axis.get("range").get("use_default"), key="{0}_axis_range_use_default".format(label)) + if not use_default_ranges: + axis_range_min, axis_range_max = st.columns(2) + with axis_range_min: + st.number_input("Minimum", key="{0}_axis_range_min".format(label)) + with axis_range_max: + st.number_input("Maximum", key="{0}_axis_range_max".format(label)) + # scaling select if label == "y": st.write("---") @@ -294,6 +304,9 @@ def update_axes(): x_column = state.x_axis_column x_units_column = state.x_axis_units_column x_units_custom = state.x_axis_units_custom + x_range_use_default = state.x_axis_range_use_default + x_range_min = None if x_range_use_default else state.x_axis_range_min + x_range_max = None if x_range_use_default else state.x_axis_range_max y_column = state.y_axis_column y_units_column = state.y_axis_units_column @@ -302,6 +315,9 @@ def update_axes(): y_scaling_series = state.y_axis_scaling_series y_scaling_x = state.y_axis_scaling_x_value y_scaling_custom = state.y_axis_custom_scaling_val + y_range_use_default = state.y_axis_range_use_default + y_range_min = None if y_range_use_default else state.y_axis_range_min + y_range_max = None if y_range_use_default else state.y_axis_range_max # update columns config.x_axis["value"] = x_column @@ -321,6 +337,13 @@ def update_axes(): config.y_axis["units"] = {"column": y_units_column} config.column_types[y_units_column] = "str" + config.x_axis["range"]["use_default"] = x_range_use_default + config.x_axis["range"]["min"] = x_range_min + config.x_axis["range"]["max"] = x_range_max + config.y_axis["range"]["use_default"] = y_range_use_default + config.y_axis["range"]["min"] = y_range_min + config.y_axis["range"]["max"] = y_range_max + # update scaling config.y_axis["scaling"] = {"custom": y_scaling_custom if y_scaling_custom else None} if not y_scaling_custom and y_scaling_column: diff --git a/post-processing/test_post_processing.py b/post-processing/test_post_processing.py index d595bae8..6faadc92 100644 --- a/post-processing/test_post_processing.py +++ b/post-processing/test_post_processing.py @@ -226,11 +226,16 @@ def test_high_level_script(run_sombrero): try: PostProcessing(sombrero_log_path).run_post_processing( ConfigHandler( - {"title": "Title", + {"plot_type": "generic", + "title": "Title", "x_axis": {"value": "tasks", - "units": {"custom": None}}, + "units": {"custom": None}, + "range": {"use_default": True, + "min": None, "max": None}}, "y_axis": {"value": "flops_value", - "units": {"column": "flops_unit"}}, + "units": {"column": "flops_unit"}, + "range": {"use_default": True, + "min": None, "max": None}}, "filters": {"and": [["fake_column", "==", 2]], "or": []}, "series": [], @@ -247,11 +252,16 @@ def test_high_level_script(run_sombrero): try: PostProcessing(sombrero_log_path).run_post_processing( ConfigHandler( - {"title": "Title", + {"plot_type": "generic", + "title": "Title", "x_axis": {"value": "tasks", - "units": {"custom": None}}, + "units": {"custom": None}, + "range": {"use_default": True, + "min": None, "max": None}}, "y_axis": {"value": "flops_value", - "units": {"column": "flops_unit"}}, + "units": {"column": "flops_unit"}, + "range": {"use_default": True, + "min": None, "max": None}}, "filters": {"and": [["tasks", "!!", 2]], "or": []}, "series": [], @@ -268,11 +278,16 @@ def test_high_level_script(run_sombrero): try: PostProcessing(sombrero_log_path).run_post_processing( ConfigHandler( - {"title": "Title", + {"plot_type": "generic", + "title": "Title", "x_axis": {"value": "tasks", - "units": {"custom": None}}, + "units": {"custom": None}, + "range": {"use_default": True, + "min": None, "max": None}}, "y_axis": {"value": "flops_value", - "units": {"column": "flops_unit"}}, + "units": {"column": "flops_unit"}, + "range": {"use_default": True, + "min": None, "max": None}}, "filters": {"and": [["flops_value", ">", "v"]], "or": []}, "series": [], @@ -289,11 +304,16 @@ def test_high_level_script(run_sombrero): try: PostProcessing(sombrero_log_path).run_post_processing( ConfigHandler( - {"title": "Title", + {"plot_type": "generic", + "title": "Title", "x_axis": {"value": "tasks", - "units": {"custom": None}}, + "units": {"custom": None}, + "range": {"use_default": True, + "min": None, "max": None}}, "y_axis": {"value": "flops_value", - "units": {"column": "flops_unit"}}, + "units": {"column": "flops_unit"}, + "range": {"use_default": True, + "min": None, "max": None}}, "filters": {"and": [["tasks", ">", 2]], "or": []}, "series": [], @@ -310,11 +330,16 @@ def test_high_level_script(run_sombrero): try: df = PostProcessing(sombrero_log_path).run_post_processing( ConfigHandler( - {"title": "Title", + {"plot_type": "generic", + "title": "Title", "x_axis": {"value": "tasks", - "units": {"custom": None}}, + "units": {"custom": None}, + "range": {"use_default": True, + "min": None, "max": None}}, "y_axis": {"value": "flops_value", - "units": {"column": "flops_unit"}}, + "units": {"column": "flops_unit"}, + "range": {"use_default": True, + "min": None, "max": None}}, "filters": {"and": [], "or": []}, "series": [], "column_types": {"tasks": "int", @@ -330,11 +355,16 @@ def test_high_level_script(run_sombrero): try: df = PostProcessing(sombrero_changed_log_path).run_post_processing( ConfigHandler( - {"title": "Title", + {"plot_type": "generic", + "title": "Title", "x_axis": {"value": "tasks", - "units": {"custom": None}}, + "units": {"custom": None}, + "range": {"use_default": True, + "min": None, "max": None}}, "y_axis": {"value": "cpus_per_task", - "units": {"column": "extra_param"}}, + "units": {"column": "extra_param"}, + "range": {"use_default": True, + "min": None, "max": None}}, "filters": {"and": [], "or": []}, "series": [], "column_types": {"tasks": "int", @@ -352,11 +382,16 @@ def test_high_level_script(run_sombrero): # check correct date filtering df = PostProcessing(sombrero_changed_log_path).run_post_processing( ConfigHandler( - {"title": "Title", + {"plot_type": "generic", + "title": "Title", "x_axis": {"value": "job_completion_time", - "units": {"custom": None}}, + "units": {"custom": None}, + "range": {"use_default": True, + "min": None, "max": None}}, "y_axis": {"value": "flops_value", - "units": {"column": "flops_unit"}}, + "units": {"column": "flops_unit"}, + "range": {"use_default": True, + "min": None, "max": None}}, "filters": {"and": [["job_completion_time", ">", "2000-06-01T12:30:15"]], "or": []}, "series": [], @@ -370,11 +405,16 @@ def test_high_level_script(run_sombrero): # check correct or filtering df = PostProcessing(sombrero_log_path).run_post_processing( ConfigHandler( - {"title": "Title", + {"plot_type": "generic", + "title": "Title", "x_axis": {"value": "tasks", - "units": {"custom": None}}, + "units": {"custom": None}, + "range": {"use_default": True, + "min": None, "max": None}}, "y_axis": {"value": "flops_value", - "units": {"column": "flops_unit"}}, + "units": {"column": "flops_unit"}, + "range": {"use_default": True, + "min": None, "max": None}}, "filters": {"and": [], "or": [["tasks", ">", "1"], ["tasks", "<", "2"]]}, "series": [["cpus_per_task", "1"], ["cpus_per_task", "2"]], @@ -389,12 +429,17 @@ def test_high_level_script(run_sombrero): # check correct column scaling dfs = PostProcessing(sombrero_log_path).run_post_processing( ConfigHandler( - {"title": "Title", + {"plot_type": "generic", + "title": "Title", "x_axis": {"value": "tasks", - "units": {"custom": None}}, + "units": {"custom": None}, + "range": {"use_default": True, + "min": None, "max": None}}, "y_axis": {"value": "flops_value", "units": {"column": "flops_unit"}, - "scaling": {"column": {"name": "OMP_NUM_THREADS"}}}, + "scaling": {"column": {"name": "OMP_NUM_THREADS"}}, + "range": {"use_default": True, + "min": None, "max": None}}, "filters": {"and": [["cpus_per_task", "==", 2]], "or": []}, "series": [], @@ -410,13 +455,18 @@ def test_high_level_script(run_sombrero): # check correct column + series scaling dfs = PostProcessing(sombrero_log_path).run_post_processing( ConfigHandler( - {"title": "Title", + {"plot_type": "generic", + "title": "Title", "x_axis": {"value": "tasks", - "units": {"custom": None}}, + "units": {"custom": None}, + "range": {"use_default": True, + "min": None, "max": None}}, "y_axis": {"value": "flops_value", "units": {"column": "flops_unit"}, "scaling": {"column": {"name": "flops_value", - "series": 0}}}, + "series": 0}}, + "range": {"use_default": True, + "min": None, "max": None}}, "filters": {"and": [], "or": []}, "series": [["cpus_per_task", 1], ["cpus_per_task", 2]], "column_types": {"tasks": "int", @@ -434,14 +484,19 @@ def test_high_level_script(run_sombrero): # check correct column + series + x value scaling dfs = PostProcessing(sombrero_log_path).run_post_processing( ConfigHandler( - {"title": "Title", + {"plot_type": "generic", + "title": "Title", "x_axis": {"value": "tasks", - "units": {"custom": None}}, + "units": {"custom": None}, + "range": {"use_default": True, + "min": None, "max": None}}, "y_axis": {"value": "flops_value", "units": {"column": "flops_unit"}, "scaling": {"column": {"name": "flops_value", "series": 0, - "x_value": 2}}}, + "x_value": 2}}, + "range": {"use_default": True, + "min": None, "max": None}}, "filters": {"and": [], "or": []}, "series": [["cpus_per_task", 1], ["cpus_per_task", 2]], "column_types": {"tasks": "int", @@ -455,12 +510,17 @@ def test_high_level_script(run_sombrero): # check correct custom scaling dfs = PostProcessing(sombrero_log_path).run_post_processing( ConfigHandler( - {"title": "Title", + {"plot_type": "generic", + "title": "Title", "x_axis": {"value": "tasks", - "units": {"custom": None}}, + "units": {"custom": None}, + "range": {"use_default": True, + "min": None, "max": None}}, "y_axis": {"value": "flops_value", "units": {"column": "flops_unit"}, - "scaling": {"custom": 2}}, + "scaling": {"custom": 2}, + "range": {"use_default": True, + "min": None, "max": None}}, "filters": {"and": [["cpus_per_task", "==", 2]], "or": []}, "series": [], @@ -476,12 +536,17 @@ def test_high_level_script(run_sombrero): try: df = PostProcessing(sombrero_log_path).run_post_processing( ConfigHandler( - {"title": "Title", + {"plot_type": "generic", + "title": "Title", "x_axis": {"value": "tasks", - "units": {"custom": None}}, + "units": {"custom": None}, + "range": {"use_default": True, + "min": None, "max": None}}, "y_axis": {"value": "flops_value", "units": {"column": "flops_unit"}, - "scaling": {"column": {"name": "OMP_NUM_THREADS"}}}, + "scaling": {"column": {"name": "OMP_NUM_THREADS"}}, + "range": {"use_default": True, + "min": None, "max": None}}, "filters": {"and": [], "or": []}, "series": [["cpus_per_task", 1], ["cpus_per_task", 2]], "column_types": {"tasks": "int", @@ -497,12 +562,17 @@ def test_high_level_script(run_sombrero): try: df = PostProcessing(sombrero_log_path).run_post_processing( ConfigHandler( - {"title": "Title", + {"plot_type": "generic", + "title": "Title", "x_axis": {"value": "tasks", - "units": {"custom": None}}, + "units": {"custom": None}, + "range": {"use_default": True, + "min": None, "max": None}}, "y_axis": {"value": "flops_value", "units": {"column": "flops_unit"}, - "scaling": {"custom": "s"}}, + "scaling": {"custom": "s"}, + "range": {"use_default": True, + "min": None, "max": None}}, "filters": {"and": [["cpus_per_task", "==", 2]], "or": []}, "series": [], @@ -519,11 +589,16 @@ def test_high_level_script(run_sombrero): # get collated dataframe subset df = PostProcessing(Path(sombrero_log_path).parent).run_post_processing( ConfigHandler( - {"title": "Title", + {"plot_type": "generic", + "title": "Title", "x_axis": {"value": "tasks", - "units": {"custom": None}}, + "units": {"custom": None}, + "range": {"use_default": True, + "min": None, "max": None}}, "y_axis": {"value": "flops_value", - "units": {"column": "flops_unit"}}, + "units": {"column": "flops_unit"}, + "range": {"use_default": True, + "min": None, "max": None}}, "filters": {"and": [], "or": []}, "series": [], "column_types": {"tasks": "int", @@ -539,11 +614,16 @@ def test_high_level_script(run_sombrero): # get filtered dataframe subset df = PostProcessing(sombrero_log_path).run_post_processing( ConfigHandler( - {"title": "Title", + {"plot_type": "generic", + "title": "Title", "x_axis": {"value": "tasks", - "units": {"custom": None}}, + "units": {"custom": None}, + "range": {"use_default": True, + "min": None, "max": None}}, "y_axis": {"value": "flops_value", - "units": {"column": "flops_unit"}}, + "units": {"column": "flops_unit"}, + "range": {"use_default": True, + "min": None, "max": None}}, "filters": {"and": [["tasks", ">", 1], ["cpus_per_task", "==", 2]], "or": []}, "series": [], @@ -561,11 +641,16 @@ def test_high_level_script(run_sombrero): # get filtered dataframe with extra columns for csv df = PostProcessing(sombrero_log_path, save=True).run_post_processing( ConfigHandler( - {"title": "Title", + {"plot_type": "generic", + "title": "Title", "x_axis": {"value": "tasks", - "units": {"custom": None}}, + "units": {"custom": None}, + "range": {"use_default": True, + "min": None, "max": None}}, "y_axis": {"value": "flops_value", - "units": {"column": "flops_unit"}}, + "units": {"column": "flops_unit"}, + "range": {"use_default": True, + "min": None, "max": None}}, "filters": {"and": [["tasks", ">", 1], ["cpus_per_task", "==", 2]], "or": []}, "series": [], @@ -591,11 +676,16 @@ def test_high_level_script(run_sombrero): # get filtered dataframe with duplicated extra columns for csv df = PostProcessing(sombrero_log_path, save=True).run_post_processing( ConfigHandler( - {"title": "Title", + {"plot_type": "generic", + "title": "Title", "x_axis": {"value": "tasks", - "units": {"custom": None}}, + "units": {"custom": None}, + "range": {"use_default": True, + "min": None, "max": None}}, "y_axis": {"value": "flops_value", - "units": {"column": "flops_unit"}}, + "units": {"column": "flops_unit"}, + "range": {"use_default": True, + "min": None, "max": None}}, "filters": {"and": [["tasks", ">", 1], ["cpus_per_task", "==", 2]], "or": []}, "series": [], From 213b3290448d650c4bea20a7b817e5ac4febdf59 Mon Sep 17 00:00:00 2001 From: Connor Aird Date: Fri, 24 May 2024 17:49:11 +0100 Subject: [PATCH 5/8] Use more specific check for if ranges are truthy --- post-processing/plot_handler.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/post-processing/plot_handler.py b/post-processing/plot_handler.py index 89fc65f0..d11cdd7f 100644 --- a/post-processing/plot_handler.py +++ b/post-processing/plot_handler.py @@ -27,12 +27,11 @@ def get_axis_min_max(df, axis, column): buffer_time = datetime_range*0.2 axis_min = axis_min_element - buffer_time axis_max = axis_max_element + buffer_time - else: - if not (axis_min and axis_max): - axis_min = (axis_min_element*0.6 if min(df[column]) >= 0 - else math.floor(axis_min_element*1.2)) - axis_max = (axis_max_element*0.6 if max(df[column]) <= 0 - else math.ceil(axis_max_element*1.2)) + elif axis_min is None or axis_max is None: + axis_min = (axis_min_element*0.6 if min(df[column]) >= 0 + else math.floor(axis_min_element*1.2)) + axis_max = (axis_max_element*0.6 if max(df[column]) <= 0 + else math.ceil(axis_max_element*1.2)) return axis_min, axis_max From bee56dbaafeff031d69c0d5e17bd5a8af87cd641 Mon Sep 17 00:00:00 2001 From: Connor Aird Date: Tue, 28 May 2024 11:05:39 +0100 Subject: [PATCH 6/8] Make generic to default plot_type and hide custom range elements if plot_type != 'line' --- post-processing/streamlit_post_processing.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/post-processing/streamlit_post_processing.py b/post-processing/streamlit_post_processing.py index 7d9988a3..7a6eb08f 100644 --- a/post-processing/streamlit_post_processing.py +++ b/post-processing/streamlit_post_processing.py @@ -70,7 +70,7 @@ def update_ui(post: PostProcessing, config: ConfigHandler, e: 'Exception | None' # set plot type plot_type_options = ['generic', 'line'] - plot_type_index = plot_type_options.index(config.plot_type) if config.plot_type else None + plot_type_index = plot_type_options.index(config.plot_type) if config.plot_type else 0 plot_type = st.selectbox("#### Plot type", plot_type_options, key="plot_type", index=plot_type_index) if plot_type != config.plot_type: @@ -136,18 +136,18 @@ def axis_options(): with st.container(border=True): # x-axis select - axis_select("x", config.x_axis) + axis_select("x", config.x_axis, config.plot_type) sort = st.checkbox("sort descending", True if config.x_axis.get("sort") == "descending" else False) with st.container(border=True): # y-axis select - axis_select("y", config.y_axis) + axis_select("y", config.y_axis, config.plot_type) # apply changes update_axes() config.x_axis["sort"] = "descending" if sort else "ascending" -def axis_select(label: str, axis: dict): +def axis_select(label: str, axis: dict, plot_type: str): """ Allow the user to select axis column and type for post-processing. @@ -178,8 +178,10 @@ def axis_select(label: str, axis: dict): # units select units_select(label, axis) - # axis range - use_default_ranges = st.checkbox("Use default ranges", axis.get("range").get("use_default"), key="{0}_axis_range_use_default".format(label)) + #FIXME: add ability to use a custom value for only one of min or max + disable_user_range = plot_type != 'line' + use_default_ranges = st.checkbox("Use default ranges", axis.get("range").get("use_default") if not disable_user_range else True, key="{0}_axis_range_use_default".format(label), + disabled=disable_user_range, help="Note: Custom ranges are not implemented for datetime types") if not use_default_ranges: axis_range_min, axis_range_max = st.columns(2) with axis_range_min: From a2b780b9aaeac09b0128bd8dfd5ea9fc56a337bb Mon Sep 17 00:00:00 2001 From: pineapple-cat Date: Wed, 29 May 2024 19:30:17 +0100 Subject: [PATCH 7/8] Style, consistency, and help info clarification update. --- post-processing/config_handler.py | 7 +- post-processing/plot_handler.py | 195 ++++++++++--------- post-processing/streamlit_post_processing.py | 19 +- post-processing/test_post_processing.py | 38 ++-- 4 files changed, 140 insertions(+), 119 deletions(-) diff --git a/post-processing/config_handler.py b/post-processing/config_handler.py index 2a62b1dc..0e421819 100644 --- a/post-processing/config_handler.py +++ b/post-processing/config_handler.py @@ -63,8 +63,11 @@ def from_template(self): return self(dict({ "plot_type": None, "title": None, - "x_axis": {"value": None, "units": {"custom": None}, "range": {"use_default": True, "min": None, "max": None}}, - "y_axis": {"value": None, "units": {"custom": None}, "scaling": {"custom": None}, "range": {"use_default": True, "min": None, "max": None}}, + "x_axis": {"value": None, "units": {"custom": None}, + "range": {"use_default": True, "min": None, "max": None}}, + "y_axis": {"value": None, "units": {"custom": None}, + "scaling": {"custom": None}, + "range": {"use_default": True, "min": None, "max": None}}, "filters": {"and": [], "or": []}, "series": [], "column_types": {}, diff --git a/post-processing/plot_handler.py b/post-processing/plot_handler.py index d11cdd7f..fdbc3115 100644 --- a/post-processing/plot_handler.py +++ b/post-processing/plot_handler.py @@ -1,102 +1,17 @@ +import itertools import math import os from pathlib import Path import numpy as np import pandas as pd -from pandas.api.types import is_datetime64_any_dtype as is_datetime from bokeh.models import HoverTool, Legend from bokeh.models.sources import ColumnDataSource from bokeh.palettes import viridis from bokeh.plotting import figure, output_file, save from bokeh.transform import factor_cmap +from pandas.api.types import is_datetime64_any_dtype as is_datetime from titlecase import titlecase -import itertools - -def get_axis_min_max(df, axis, column): - axis_range = axis["range"] - axis_min = axis_range["min"] if axis_range["min"] != 'None' else None - axis_max = axis_range["max"] if axis_range["max"] != 'None' else None - - #FIXME: str types and user defined datetime ranges not currently supported - # use defaults if type is datetime - axis_min_element = np.nanmin(df[column]) - axis_max_element = np.nanmax(df[column]) - if (is_datetime(df[column])): - datetime_range = axis_max_element - axis_min_element - buffer_time = datetime_range*0.2 - axis_min = axis_min_element - buffer_time - axis_max = axis_max_element + buffer_time - elif axis_min is None or axis_max is None: - axis_min = (axis_min_element*0.6 if min(df[column]) >= 0 - else math.floor(axis_min_element*1.2)) - axis_max = (axis_max_element*0.6 if max(df[column]) <= 0 - else math.ceil(axis_max_element*1.2)) - - return axis_min, axis_max - -def plot_line_chart(title, df: pd.DataFrame, x_axis, y_axis, series_filters): - """ - Create a line chart for the supplied data using bokeh. - - Args: - title: str, plot title. - df: dataframe, data to plot. - x_axis: dict, x-axis column and units. - y_axis: dict, y-axis column and units. - series_filters: list, x-axis groups used to filter graph data. - """ - # get column names and labels for axes - x_column, x_label = get_axis_labels(df, x_axis, series_filters) - y_column, y_label = get_axis_labels(df, y_axis, series_filters) - - min_x, max_x = get_axis_min_max(df, x_axis, x_column) - min_y, max_y = get_axis_min_max(df, y_axis, y_column) - - # create html file to store plot in - output_file(filename=os.path.join( - Path(__file__).parent, "{0}.html".format(title.replace(" ", "_"))), title=title) - - # create plot - plot = figure(x_range=(min_x, max_x), y_range=(min_y, max_y), title=title, - width=800, toolbar_location="above") - - # configure tooltip - plot.add_tools(HoverTool(tooltips=[ - (y_label, "@{0}".format(y_column) - + ("{%0.2f}" if pd.api.types.is_float_dtype(df[y_column].dtype) - else ""))], - formatters={"@{0}".format(y_column): "printf"})) - - # create legend outside plot - plot.add_layout(Legend(), "right") - - colours = itertools.cycle(viridis(len(series_filters))) - - for filter in series_filters: - filtered_df = None - if filter[1] == '==': - filtered_df = df[df[filter[0]] == int(filter[2])] - plot.line(x=x_column, y=y_column, source=filtered_df, legend_label=' '.join(filter), line_width=2, color=next(colours)) - - # add labels - plot.xaxis.axis_label = x_label - plot.yaxis.axis_label = y_label - # adjust font size - plot.title.text_font_size = "15pt" - - # flip x-axis if sort is descending - if x_axis.get("sort"): - if x_axis["sort"] == "descending": - end = plot.x_range.end - start = plot.x_range.start - plot.x_range.start = end - plot.x_range.end = start - - # save to file - save(plot) - - return plot def plot_generic(title, df: pd.DataFrame, x_axis, y_axis, series_filters, debug=False): @@ -152,10 +67,9 @@ def plot_generic(title, df: pd.DataFrame, x_axis, y_axis, series_filters, debug= plot = figure(x_range=grouped_df, y_range=(min_y, max_y), title=title, width=800, toolbar_location="above") # configure tooltip - plot.add_tools(HoverTool(tooltips=[ - (y_label, "@{0}_mean".format(y_column) - + ("{%0.2f}" if pd.api.types.is_float_dtype(df[y_column].dtype) - else ""))], + plot.add_tools(HoverTool(tooltips=[(y_label, "@{0}_mean".format(y_column) + + ("{%0.2f}" if pd.api.types.is_float_dtype(df[y_column].dtype) + else ""))], formatters={"@{0}_mean".format(y_column): "printf"})) # sort x-axis values in descending order (otherwise default sort is ascending) @@ -266,3 +180,102 @@ def get_axis_labels(df: pd.DataFrame, axis, series_filters): " ({0})".format(units) if units else "") return col_name, label + + +def plot_line_chart(title, df: pd.DataFrame, x_axis, y_axis, series_filters): + """ + Create a line chart for the supplied data using bokeh. + + Args: + title: str, plot title. + df: dataframe, data to plot. + x_axis: dict, x-axis column and units. + y_axis: dict, y-axis column and units. + series_filters: list, x-axis groups used to filter graph data. + """ + + # get column names and labels for axes + x_column, x_label = get_axis_labels(df, x_axis, series_filters) + y_column, y_label = get_axis_labels(df, y_axis, series_filters) + + # adjust axis ranges + min_x, max_x = get_axis_min_max(df, x_axis) + min_y, max_y = get_axis_min_max(df, y_axis) + + # create html file to store plot in + output_file(filename=os.path.join( + Path(__file__).parent, "{0}.html".format(title.replace(" ", "_"))), title=title) + + # create plot + plot = figure(x_range=(min_x, max_x), y_range=(min_y, max_y), title=title, + width=800, toolbar_location="above") + + # configure tooltip + plot.add_tools(HoverTool(tooltips=[(y_label, "@{0}".format(y_column) + + ("{%0.2f}" if pd.api.types.is_float_dtype(df[y_column].dtype) + else ""))], + formatters={"@{0}".format(y_column): "printf"})) + + # create legend outside plot + plot.add_layout(Legend(), "right") + colours = itertools.cycle(viridis(len(series_filters))) + + for filter in series_filters: + filtered_df = None + if filter[1] == '==': + filtered_df = df[df[filter[0]] == int(filter[2])] + plot.line(x=x_column, y=y_column, source=filtered_df, legend_label=' '.join(filter), + line_width=2, color=next(colours)) + + # add labels + plot.xaxis.axis_label = x_label + plot.yaxis.axis_label = y_label + # adjust font size + plot.title.text_font_size = "15pt" + + # flip x-axis if sort is descending + if x_axis.get("sort"): + if x_axis["sort"] == "descending": + end = plot.x_range.end + start = plot.x_range.start + plot.x_range.start = end + plot.x_range.end = start + + # save to file + save(plot) + + return plot + + +def get_axis_min_max(df, axis): + """ + Return the minimum and maximum numeric values for a given axis. + + Args: + df: dataframe, data to plot. + axis: dict, axis column, units, and values to scale by. + """ + + column = axis["value"] + axis_range = axis["range"] + axis_min = axis_range["min"] if axis_range["min"] != 'None' else None + axis_max = axis_range["max"] if axis_range["max"] != 'None' else None + + # FIXME: str types and user defined datetime ranges not currently supported + axis_min_element = np.nanmin(df[column]) + axis_max_element = np.nanmax(df[column]) + + # use defaults if type is datetime + if (is_datetime(df[column])): + datetime_range = axis_max_element - axis_min_element + buffer_time = datetime_range*0.2 + axis_min = axis_min_element - buffer_time + axis_max = axis_max_element + buffer_time + + elif axis_min is None or axis_max is None: + axis_min = (axis_min_element*0.6 if min(df[column]) >= 0 + else math.floor(axis_min_element*1.2)) + axis_max = (axis_max_element*0.6 if max(df[column]) <= 0 + else math.ceil(axis_max_element*1.2)) + + return axis_min, axis_max diff --git a/post-processing/streamlit_post_processing.py b/post-processing/streamlit_post_processing.py index 7a6eb08f..d947f810 100644 --- a/post-processing/streamlit_post_processing.py +++ b/post-processing/streamlit_post_processing.py @@ -72,7 +72,7 @@ def update_ui(post: PostProcessing, config: ConfigHandler, e: 'Exception | None' plot_type_options = ['generic', 'line'] plot_type_index = plot_type_options.index(config.plot_type) if config.plot_type else 0 plot_type = st.selectbox("#### Plot type", plot_type_options, - key="plot_type", index=plot_type_index) + key="plot_type", index=plot_type_index) if plot_type != config.plot_type: config.plot_type = plot_type @@ -177,17 +177,22 @@ def axis_select(label: str, axis: dict, plot_type: str): # units select units_select(label, axis) - - #FIXME: add ability to use a custom value for only one of min or max + + # FIXME: add ability to use a custom value for only one of min or max disable_user_range = plot_type != 'line' - use_default_ranges = st.checkbox("Use default ranges", axis.get("range").get("use_default") if not disable_user_range else True, key="{0}_axis_range_use_default".format(label), - disabled=disable_user_range, help="Note: Custom ranges are not implemented for datetime types") + use_default_ranges = st.checkbox("default axis range", + axis.get("range").get("use_default") if not disable_user_range else True, + key="{0}_axis_range_use_default".format(label), + disabled=disable_user_range, + help="{0} {1}".format( + "Uncheck to assign custom minimum and maximum values to axis.", + "Custom ranges are not implemented for datetime types.")) if not use_default_ranges: axis_range_min, axis_range_max = st.columns(2) with axis_range_min: - st.number_input("Minimum", key="{0}_axis_range_min".format(label)) + st.number_input("{0}-axis minimum".format(label), key="{0}_axis_range_min".format(label)) with axis_range_max: - st.number_input("Maximum", key="{0}_axis_range_max".format(label)) + st.number_input("{0}-axis maximum".format(label), key="{0}_axis_range_max".format(label)) # scaling select if label == "y": diff --git a/post-processing/test_post_processing.py b/post-processing/test_post_processing.py index 6faadc92..89a591f0 100644 --- a/post-processing/test_post_processing.py +++ b/post-processing/test_post_processing.py @@ -387,11 +387,11 @@ def test_high_level_script(run_sombrero): "x_axis": {"value": "job_completion_time", "units": {"custom": None}, "range": {"use_default": True, - "min": None, "max": None}}, + "min": None, "max": None}}, "y_axis": {"value": "flops_value", "units": {"column": "flops_unit"}, "range": {"use_default": True, - "min": None, "max": None}}, + "min": None, "max": None}}, "filters": {"and": [["job_completion_time", ">", "2000-06-01T12:30:15"]], "or": []}, "series": [], @@ -410,11 +410,11 @@ def test_high_level_script(run_sombrero): "x_axis": {"value": "tasks", "units": {"custom": None}, "range": {"use_default": True, - "min": None, "max": None}}, + "min": None, "max": None}}, "y_axis": {"value": "flops_value", "units": {"column": "flops_unit"}, "range": {"use_default": True, - "min": None, "max": None}}, + "min": None, "max": None}}, "filters": {"and": [], "or": [["tasks", ">", "1"], ["tasks", "<", "2"]]}, "series": [["cpus_per_task", "1"], ["cpus_per_task", "2"]], @@ -434,12 +434,12 @@ def test_high_level_script(run_sombrero): "x_axis": {"value": "tasks", "units": {"custom": None}, "range": {"use_default": True, - "min": None, "max": None}}, + "min": None, "max": None}}, "y_axis": {"value": "flops_value", "units": {"column": "flops_unit"}, "scaling": {"column": {"name": "OMP_NUM_THREADS"}}, "range": {"use_default": True, - "min": None, "max": None}}, + "min": None, "max": None}}, "filters": {"and": [["cpus_per_task", "==", 2]], "or": []}, "series": [], @@ -460,13 +460,13 @@ def test_high_level_script(run_sombrero): "x_axis": {"value": "tasks", "units": {"custom": None}, "range": {"use_default": True, - "min": None, "max": None}}, + "min": None, "max": None}}, "y_axis": {"value": "flops_value", "units": {"column": "flops_unit"}, "scaling": {"column": {"name": "flops_value", "series": 0}}, "range": {"use_default": True, - "min": None, "max": None}}, + "min": None, "max": None}}, "filters": {"and": [], "or": []}, "series": [["cpus_per_task", 1], ["cpus_per_task", 2]], "column_types": {"tasks": "int", @@ -489,14 +489,14 @@ def test_high_level_script(run_sombrero): "x_axis": {"value": "tasks", "units": {"custom": None}, "range": {"use_default": True, - "min": None, "max": None}}, + "min": None, "max": None}}, "y_axis": {"value": "flops_value", "units": {"column": "flops_unit"}, "scaling": {"column": {"name": "flops_value", "series": 0, "x_value": 2}}, "range": {"use_default": True, - "min": None, "max": None}}, + "min": None, "max": None}}, "filters": {"and": [], "or": []}, "series": [["cpus_per_task", 1], ["cpus_per_task", 2]], "column_types": {"tasks": "int", @@ -515,12 +515,12 @@ def test_high_level_script(run_sombrero): "x_axis": {"value": "tasks", "units": {"custom": None}, "range": {"use_default": True, - "min": None, "max": None}}, + "min": None, "max": None}}, "y_axis": {"value": "flops_value", "units": {"column": "flops_unit"}, "scaling": {"custom": 2}, "range": {"use_default": True, - "min": None, "max": None}}, + "min": None, "max": None}}, "filters": {"and": [["cpus_per_task", "==", 2]], "or": []}, "series": [], @@ -619,11 +619,11 @@ def test_high_level_script(run_sombrero): "x_axis": {"value": "tasks", "units": {"custom": None}, "range": {"use_default": True, - "min": None, "max": None}}, + "min": None, "max": None}}, "y_axis": {"value": "flops_value", "units": {"column": "flops_unit"}, "range": {"use_default": True, - "min": None, "max": None}}, + "min": None, "max": None}}, "filters": {"and": [["tasks", ">", 1], ["cpus_per_task", "==", 2]], "or": []}, "series": [], @@ -642,15 +642,15 @@ def test_high_level_script(run_sombrero): df = PostProcessing(sombrero_log_path, save=True).run_post_processing( ConfigHandler( {"plot_type": "generic", - "title": "Title", + "title": "Title", "x_axis": {"value": "tasks", "units": {"custom": None}, "range": {"use_default": True, - "min": None, "max": None}}, + "min": None, "max": None}}, "y_axis": {"value": "flops_value", "units": {"column": "flops_unit"}, "range": {"use_default": True, - "min": None, "max": None}}, + "min": None, "max": None}}, "filters": {"and": [["tasks", ">", 1], ["cpus_per_task", "==", 2]], "or": []}, "series": [], @@ -681,11 +681,11 @@ def test_high_level_script(run_sombrero): "x_axis": {"value": "tasks", "units": {"custom": None}, "range": {"use_default": True, - "min": None, "max": None}}, + "min": None, "max": None}}, "y_axis": {"value": "flops_value", "units": {"column": "flops_unit"}, "range": {"use_default": True, - "min": None, "max": None}}, + "min": None, "max": None}}, "filters": {"and": [["tasks", ">", 1], ["cpus_per_task", "==", 2]], "or": []}, "series": [], From a26123e509e854cbff839a0fbfc8c1d6a87ed8bc Mon Sep 17 00:00:00 2001 From: Connor Aird Date: Wed, 5 Jun 2024 11:18:52 +0100 Subject: [PATCH 8/8] remove use_defaults buttons and auto fill ranges with values if the user doesn't provide any --- post-processing/config_handler.py | 4 +- post-processing/plot_handler.py | 35 +++--- post-processing/streamlit_post_processing.py | 37 +++---- post-processing/test_post_processing.py | 108 +++++++------------ 4 files changed, 71 insertions(+), 113 deletions(-) diff --git a/post-processing/config_handler.py b/post-processing/config_handler.py index 0e421819..cdef1ec3 100644 --- a/post-processing/config_handler.py +++ b/post-processing/config_handler.py @@ -64,10 +64,10 @@ def from_template(self): "plot_type": None, "title": None, "x_axis": {"value": None, "units": {"custom": None}, - "range": {"use_default": True, "min": None, "max": None}}, + "range": {"min": None, "max": None}}, "y_axis": {"value": None, "units": {"custom": None}, "scaling": {"custom": None}, - "range": {"use_default": True, "min": None, "max": None}}, + "range": {"min": None, "max": None}}, "filters": {"and": [], "or": []}, "series": [], "column_types": {}, diff --git a/post-processing/plot_handler.py b/post-processing/plot_handler.py index fdbc3115..5b2d11b4 100644 --- a/post-processing/plot_handler.py +++ b/post-processing/plot_handler.py @@ -258,24 +258,25 @@ def get_axis_min_max(df, axis): column = axis["value"] axis_range = axis["range"] - axis_min = axis_range["min"] if axis_range["min"] != 'None' else None - axis_max = axis_range["max"] if axis_range["max"] != 'None' else None + axis_min = axis_range["min"] if axis_range["min"] else 0.0 + axis_max = axis_range["max"] if axis_range["max"] else 0.0 # FIXME: str types and user defined datetime ranges not currently supported - axis_min_element = np.nanmin(df[column]) - axis_max_element = np.nanmax(df[column]) - - # use defaults if type is datetime - if (is_datetime(df[column])): - datetime_range = axis_max_element - axis_min_element - buffer_time = datetime_range*0.2 - axis_min = axis_min_element - buffer_time - axis_max = axis_max_element + buffer_time - - elif axis_min is None or axis_max is None: - axis_min = (axis_min_element*0.6 if min(df[column]) >= 0 - else math.floor(axis_min_element*1.2)) - axis_max = (axis_max_element*0.6 if max(df[column]) <= 0 - else math.ceil(axis_max_element*1.2)) + if (column): + axis_min_element = np.nanmin(df[column]) + axis_max_element = np.nanmax(df[column]) + + # use defaults if type is datetime + if (is_datetime(df[column])): + datetime_range = axis_max_element - axis_min_element + buffer_time = datetime_range*0.2 + axis_min = axis_min_element - buffer_time + axis_max = axis_max_element + buffer_time + + elif axis_min is None or axis_max is None or axis_min == axis_max: + axis_min = (axis_min_element*0.6 if min(df[column]) >= 0 + else math.floor(axis_min_element*1.2)) + axis_max = (axis_max_element*0.6 if max(df[column]) <= 0 + else math.ceil(axis_max_element*1.2)) return axis_min, axis_max diff --git a/post-processing/streamlit_post_processing.py b/post-processing/streamlit_post_processing.py index d947f810..e32b1682 100644 --- a/post-processing/streamlit_post_processing.py +++ b/post-processing/streamlit_post_processing.py @@ -5,6 +5,7 @@ import streamlit as st from config_handler import ConfigHandler, load_config, read_config from post_processing import PostProcessing +from plot_handler import get_axis_min_max # drop-down lists operators = ["==", "!=", "<", ">", "<=", ">="] @@ -179,20 +180,16 @@ def axis_select(label: str, axis: dict, plot_type: str): units_select(label, axis) # FIXME: add ability to use a custom value for only one of min or max - disable_user_range = plot_type != 'line' - use_default_ranges = st.checkbox("default axis range", - axis.get("range").get("use_default") if not disable_user_range else True, - key="{0}_axis_range_use_default".format(label), - disabled=disable_user_range, - help="{0} {1}".format( - "Uncheck to assign custom minimum and maximum values to axis.", - "Custom ranges are not implemented for datetime types.")) - if not use_default_ranges: - axis_range_min, axis_range_max = st.columns(2) - with axis_range_min: - st.number_input("{0}-axis minimum".format(label), key="{0}_axis_range_min".format(label)) - with axis_range_max: - st.number_input("{0}-axis maximum".format(label), key="{0}_axis_range_max".format(label)) + range = get_axis_min_max(df, axis) + axis_range_min, axis_range_max = st.columns(2) + with axis_range_min: + st.number_input("{0}-axis minimum".format(label), + value=range[0], + key="{0}_axis_range_min".format(label)) + with axis_range_max: + st.number_input("{0}-axis maximum".format(label), + value=range[1], + key="{0}_axis_range_max".format(label)) # scaling select if label == "y": @@ -311,9 +308,8 @@ def update_axes(): x_column = state.x_axis_column x_units_column = state.x_axis_units_column x_units_custom = state.x_axis_units_custom - x_range_use_default = state.x_axis_range_use_default - x_range_min = None if x_range_use_default else state.x_axis_range_min - x_range_max = None if x_range_use_default else state.x_axis_range_max + x_range_min = state.x_axis_range_min + x_range_max = state.x_axis_range_max y_column = state.y_axis_column y_units_column = state.y_axis_units_column @@ -322,9 +318,8 @@ def update_axes(): y_scaling_series = state.y_axis_scaling_series y_scaling_x = state.y_axis_scaling_x_value y_scaling_custom = state.y_axis_custom_scaling_val - y_range_use_default = state.y_axis_range_use_default - y_range_min = None if y_range_use_default else state.y_axis_range_min - y_range_max = None if y_range_use_default else state.y_axis_range_max + y_range_min = state.y_axis_range_min + y_range_max = state.y_axis_range_max # update columns config.x_axis["value"] = x_column @@ -344,10 +339,8 @@ def update_axes(): config.y_axis["units"] = {"column": y_units_column} config.column_types[y_units_column] = "str" - config.x_axis["range"]["use_default"] = x_range_use_default config.x_axis["range"]["min"] = x_range_min config.x_axis["range"]["max"] = x_range_max - config.y_axis["range"]["use_default"] = y_range_use_default config.y_axis["range"]["min"] = y_range_min config.y_axis["range"]["max"] = y_range_max diff --git a/post-processing/test_post_processing.py b/post-processing/test_post_processing.py index 89a591f0..cd60f2c5 100644 --- a/post-processing/test_post_processing.py +++ b/post-processing/test_post_processing.py @@ -230,12 +230,10 @@ def test_high_level_script(run_sombrero): "title": "Title", "x_axis": {"value": "tasks", "units": {"custom": None}, - "range": {"use_default": True, - "min": None, "max": None}}, + "range": {"min": None, "max": None}}, "y_axis": {"value": "flops_value", "units": {"column": "flops_unit"}, - "range": {"use_default": True, - "min": None, "max": None}}, + "range": {"min": None, "max": None}}, "filters": {"and": [["fake_column", "==", 2]], "or": []}, "series": [], @@ -256,12 +254,10 @@ def test_high_level_script(run_sombrero): "title": "Title", "x_axis": {"value": "tasks", "units": {"custom": None}, - "range": {"use_default": True, - "min": None, "max": None}}, + "range": {"min": None, "max": None}}, "y_axis": {"value": "flops_value", "units": {"column": "flops_unit"}, - "range": {"use_default": True, - "min": None, "max": None}}, + "range": {"min": None, "max": None}}, "filters": {"and": [["tasks", "!!", 2]], "or": []}, "series": [], @@ -282,12 +278,10 @@ def test_high_level_script(run_sombrero): "title": "Title", "x_axis": {"value": "tasks", "units": {"custom": None}, - "range": {"use_default": True, - "min": None, "max": None}}, + "range": {"min": None, "max": None}}, "y_axis": {"value": "flops_value", "units": {"column": "flops_unit"}, - "range": {"use_default": True, - "min": None, "max": None}}, + "range": {"min": None, "max": None}}, "filters": {"and": [["flops_value", ">", "v"]], "or": []}, "series": [], @@ -308,12 +302,10 @@ def test_high_level_script(run_sombrero): "title": "Title", "x_axis": {"value": "tasks", "units": {"custom": None}, - "range": {"use_default": True, - "min": None, "max": None}}, + "range": {"min": None, "max": None}}, "y_axis": {"value": "flops_value", "units": {"column": "flops_unit"}, - "range": {"use_default": True, - "min": None, "max": None}}, + "range": {"min": None, "max": None}}, "filters": {"and": [["tasks", ">", 2]], "or": []}, "series": [], @@ -334,12 +326,10 @@ def test_high_level_script(run_sombrero): "title": "Title", "x_axis": {"value": "tasks", "units": {"custom": None}, - "range": {"use_default": True, - "min": None, "max": None}}, + "range": {"min": None, "max": None}}, "y_axis": {"value": "flops_value", "units": {"column": "flops_unit"}, - "range": {"use_default": True, - "min": None, "max": None}}, + "range": {"min": None, "max": None}}, "filters": {"and": [], "or": []}, "series": [], "column_types": {"tasks": "int", @@ -359,12 +349,10 @@ def test_high_level_script(run_sombrero): "title": "Title", "x_axis": {"value": "tasks", "units": {"custom": None}, - "range": {"use_default": True, - "min": None, "max": None}}, + "range": {"min": None, "max": None}}, "y_axis": {"value": "cpus_per_task", "units": {"column": "extra_param"}, - "range": {"use_default": True, - "min": None, "max": None}}, + "range": {"min": None, "max": None}}, "filters": {"and": [], "or": []}, "series": [], "column_types": {"tasks": "int", @@ -386,12 +374,10 @@ def test_high_level_script(run_sombrero): "title": "Title", "x_axis": {"value": "job_completion_time", "units": {"custom": None}, - "range": {"use_default": True, - "min": None, "max": None}}, + "range": {"min": None, "max": None}}, "y_axis": {"value": "flops_value", "units": {"column": "flops_unit"}, - "range": {"use_default": True, - "min": None, "max": None}}, + "range": {"min": None, "max": None}}, "filters": {"and": [["job_completion_time", ">", "2000-06-01T12:30:15"]], "or": []}, "series": [], @@ -409,12 +395,10 @@ def test_high_level_script(run_sombrero): "title": "Title", "x_axis": {"value": "tasks", "units": {"custom": None}, - "range": {"use_default": True, - "min": None, "max": None}}, + "range": {"min": None, "max": None}}, "y_axis": {"value": "flops_value", "units": {"column": "flops_unit"}, - "range": {"use_default": True, - "min": None, "max": None}}, + "range": {"min": None, "max": None}}, "filters": {"and": [], "or": [["tasks", ">", "1"], ["tasks", "<", "2"]]}, "series": [["cpus_per_task", "1"], ["cpus_per_task", "2"]], @@ -433,13 +417,11 @@ def test_high_level_script(run_sombrero): "title": "Title", "x_axis": {"value": "tasks", "units": {"custom": None}, - "range": {"use_default": True, - "min": None, "max": None}}, + "range": {"min": None, "max": None}}, "y_axis": {"value": "flops_value", "units": {"column": "flops_unit"}, "scaling": {"column": {"name": "OMP_NUM_THREADS"}}, - "range": {"use_default": True, - "min": None, "max": None}}, + "range": {"min": None, "max": None}}, "filters": {"and": [["cpus_per_task", "==", 2]], "or": []}, "series": [], @@ -459,14 +441,12 @@ def test_high_level_script(run_sombrero): "title": "Title", "x_axis": {"value": "tasks", "units": {"custom": None}, - "range": {"use_default": True, - "min": None, "max": None}}, + "range": {"min": None, "max": None}}, "y_axis": {"value": "flops_value", "units": {"column": "flops_unit"}, "scaling": {"column": {"name": "flops_value", "series": 0}}, - "range": {"use_default": True, - "min": None, "max": None}}, + "range": {"min": None, "max": None}}, "filters": {"and": [], "or": []}, "series": [["cpus_per_task", 1], ["cpus_per_task", 2]], "column_types": {"tasks": "int", @@ -488,15 +468,13 @@ def test_high_level_script(run_sombrero): "title": "Title", "x_axis": {"value": "tasks", "units": {"custom": None}, - "range": {"use_default": True, - "min": None, "max": None}}, + "range": {"min": None, "max": None}}, "y_axis": {"value": "flops_value", "units": {"column": "flops_unit"}, "scaling": {"column": {"name": "flops_value", "series": 0, "x_value": 2}}, - "range": {"use_default": True, - "min": None, "max": None}}, + "range": {"min": None, "max": None}}, "filters": {"and": [], "or": []}, "series": [["cpus_per_task", 1], ["cpus_per_task", 2]], "column_types": {"tasks": "int", @@ -514,13 +492,11 @@ def test_high_level_script(run_sombrero): "title": "Title", "x_axis": {"value": "tasks", "units": {"custom": None}, - "range": {"use_default": True, - "min": None, "max": None}}, + "range": {"min": None, "max": None}}, "y_axis": {"value": "flops_value", "units": {"column": "flops_unit"}, "scaling": {"custom": 2}, - "range": {"use_default": True, - "min": None, "max": None}}, + "range": {"min": None, "max": None}}, "filters": {"and": [["cpus_per_task", "==", 2]], "or": []}, "series": [], @@ -540,13 +516,11 @@ def test_high_level_script(run_sombrero): "title": "Title", "x_axis": {"value": "tasks", "units": {"custom": None}, - "range": {"use_default": True, - "min": None, "max": None}}, + "range": {"min": None, "max": None}}, "y_axis": {"value": "flops_value", "units": {"column": "flops_unit"}, "scaling": {"column": {"name": "OMP_NUM_THREADS"}}, - "range": {"use_default": True, - "min": None, "max": None}}, + "range": {"min": None, "max": None}}, "filters": {"and": [], "or": []}, "series": [["cpus_per_task", 1], ["cpus_per_task", 2]], "column_types": {"tasks": "int", @@ -566,13 +540,11 @@ def test_high_level_script(run_sombrero): "title": "Title", "x_axis": {"value": "tasks", "units": {"custom": None}, - "range": {"use_default": True, - "min": None, "max": None}}, + "range": {"min": None, "max": None}}, "y_axis": {"value": "flops_value", "units": {"column": "flops_unit"}, "scaling": {"custom": "s"}, - "range": {"use_default": True, - "min": None, "max": None}}, + "range": {"min": None, "max": None}}, "filters": {"and": [["cpus_per_task", "==", 2]], "or": []}, "series": [], @@ -593,12 +565,10 @@ def test_high_level_script(run_sombrero): "title": "Title", "x_axis": {"value": "tasks", "units": {"custom": None}, - "range": {"use_default": True, - "min": None, "max": None}}, + "range": {"min": None, "max": None}}, "y_axis": {"value": "flops_value", "units": {"column": "flops_unit"}, - "range": {"use_default": True, - "min": None, "max": None}}, + "range": {"min": None, "max": None}}, "filters": {"and": [], "or": []}, "series": [], "column_types": {"tasks": "int", @@ -618,12 +588,10 @@ def test_high_level_script(run_sombrero): "title": "Title", "x_axis": {"value": "tasks", "units": {"custom": None}, - "range": {"use_default": True, - "min": None, "max": None}}, + "range": {"min": None, "max": None}}, "y_axis": {"value": "flops_value", "units": {"column": "flops_unit"}, - "range": {"use_default": True, - "min": None, "max": None}}, + "range": {"min": None, "max": None}}, "filters": {"and": [["tasks", ">", 1], ["cpus_per_task", "==", 2]], "or": []}, "series": [], @@ -645,12 +613,10 @@ def test_high_level_script(run_sombrero): "title": "Title", "x_axis": {"value": "tasks", "units": {"custom": None}, - "range": {"use_default": True, - "min": None, "max": None}}, + "range": {"min": None, "max": None}}, "y_axis": {"value": "flops_value", "units": {"column": "flops_unit"}, - "range": {"use_default": True, - "min": None, "max": None}}, + "range": {"min": None, "max": None}}, "filters": {"and": [["tasks", ">", 1], ["cpus_per_task", "==", 2]], "or": []}, "series": [], @@ -680,12 +646,10 @@ def test_high_level_script(run_sombrero): "title": "Title", "x_axis": {"value": "tasks", "units": {"custom": None}, - "range": {"use_default": True, - "min": None, "max": None}}, + "range": {"min": None, "max": None}}, "y_axis": {"value": "flops_value", "units": {"column": "flops_unit"}, - "range": {"use_default": True, - "min": None, "max": None}}, + "range": {"min": None, "max": None}}, "filters": {"and": [["tasks", ">", 1], ["cpus_per_task", "==", 2]], "or": []}, "series": [],