Merge pull request #310 from ukri-excalibur/post-processing_ui-displa…

…y-columns-fix

Integrate extra columns into Streamlit UI for post-processing

post-processing/README.md

@@ -68,7 +68,7 @@ Before running post-processing, create a config file including all necessary inf
     - `Format: [column_name, value]`
 - `column_types` - Pandas dtype for each relevant column (axes, units, filters, series). Specified with a dictionary.
     - `Accepted types: "str"/"string"/"object", "int"/"int64", "float"/"float64", "datetime"/"datetime64"`
-- `additional_columns_to_csv` - (Optional.) List of additional columns to export to csv file, in addition to the ones above. Those columns are not used in plotting. (Specify an empty list if no additional columns are required.)
+- `extra_columns_to_csv` - (Optional.) List of additional columns to include when exporting benchmark data to a CSV, in addition to the ones above. These columns are not used in plotting. (Specify an empty list if no additional columns are required.)
 
 #### A Note on Replaced ReFrame Columns
 
@@ -123,9 +123,8 @@ series: <series_list>
 column_types:
   <column_name>: <column_type>
 
-# optional (default: no extra columns exported to a csv file in addition to the ones above)
-additional_columns_to_csv:
-  <columns_list>
+# optional (default: no extra columns exported to CSV file in addition to the ones above)
+extra_columns_to_csv: <columns_list>
 ```
 
 #### Example Config

post-processing/config_handler.py

@@ -25,7 +25,7 @@ def __init__(self, config: dict, template=False):
         self.filters = config.get("filters")
         self.series = config.get("series")
         self.column_types = config.get("column_types")
-        self.extra_columns = config.get("additional_columns_to_csv")
+        self.extra_columns = config.get("extra_columns_to_csv")
 
         # parse filter information
         self.and_filters = []
@@ -65,7 +65,8 @@ def from_template(self):
             "y_axis": {"value": None, "units": {"custom": None}},
             "filters": {"and": [], "or": []},
             "series": [],
-            "column_types": {}}), template=True)
+            "column_types": {},
+            "extra_columns_to_csv": []}), template=True)
 
     def get_filters(self):
         """
@@ -144,6 +145,10 @@ def parse_columns(self):
         # drop None values
         self.plot_columns = list(dict.fromkeys([c for c in self.plot_columns if c is not None]))
 
+        # extra columns
+        if self.extra_columns is None:
+            self.extra_columns = []
+
         # filter columns
         self.filter_columns = (list(dict.fromkeys([f[0] for f in self.and_filters] +
                                                   [f[0] for f in self.or_filters]))
@@ -182,7 +187,8 @@ def to_dict(self):
             "y_axis": self.y_axis,
             "filters": self.filters,
             "series": self.series,
-            "column_types": self.column_types})
+            "column_types": self.column_types,
+            "extra_columns_to_csv": self.extra_columns})
 
     def to_yaml(self):
         """

post-processing/post_processing.py

@@ -67,8 +67,9 @@ def run_post_processing(self, config: ConfigHandler):
             print("Selected dataframe:")
             print(self.df[self.mask][config.plot_columns + config.extra_columns])
         if self.save:
+            # set index=False to exclude the dataframe index from the csv
             self.df[self.mask][config.plot_columns + config.extra_columns].to_csv(
-                path_or_buf=os.path.join(Path(__file__).parent,'output.csv'), index=True)  # Set index=False to exclude the DataFrame index from the CSV
+                path_or_buf=os.path.join(Path(__file__).parent, 'output.csv'), index=True)
 
         # call a plotting script
         if self.plotting:
@@ -406,8 +407,7 @@ def read_args():
     parser.add_argument("-s", "--save", action="store_true",
                         help="save flag for saving the filtered dataframe in csv file")
     parser.add_argument("-np", "--no_plot", action="store_true",
-                        help="no-plot flag for disabling generating and storing a plot")
-
+                        help="no-plot flag for disabling plotting")
 
     return parser.parse_args()
 
@@ -417,7 +417,7 @@ def main():
     args = read_args()
 
     try:
-        post = PostProcessing(args.log_path, args.debug, args.verbose, args.save, not(args.no_plot))
+        post = PostProcessing(args.log_path, args.debug, args.verbose, args.save, not args.no_plot)
         config = ConfigHandler.from_path(args.config_path)
         post.run_post_processing(config)
 

post-processing/streamlit_post_processing.py

@@ -46,8 +46,9 @@ def update_ui(post: PostProcessing, config: ConfigHandler, e: 'Exception | None'
     # display dataframe data
     show_df = st.toggle("Show DataFrame")
     if show_df:
-        if len(config.plot_columns) > 0:
-            st.dataframe(post.df[post.mask][config.plot_columns], hide_index=True, use_container_width=True)
+        if len(config.plot_columns + config.extra_columns) > 0:
+            st.dataframe(post.df[post.mask][config.plot_columns + config.extra_columns],
+                         hide_index=True, use_container_width=True)
         else:
             st.dataframe(post.df[post.mask], hide_index=True, use_container_width=True)
 
@@ -266,6 +267,10 @@ def filter_options():
     current_filters()
     # display new filter addition options
     new_filter_options()
+    # display current extra columns
+    extra_columns()
+    # display new extra column addition options
+    new_extra_column_options()
 
 
 def current_filters():
@@ -386,6 +391,63 @@ def add_filter(filter: list):
             update_types()
 
 
+def extra_columns():
+    """
+        Display current extra columns.
+    """
+
+    config = st.session_state.config
+    st.write("###### Current Extra Columns")
+    st.multiselect("Extra Columns", config.extra_columns if config.extra_columns else [None],
+                   config.extra_columns, key="extra_columns", on_change=update_extra_column,
+                   placeholder="None", label_visibility="collapsed")
+
+
+def new_extra_column_options():
+    """
+        Display new extra column addition options interface.
+    """
+
+    state = st.session_state
+    post = state.post
+    st.write("###### Add New Extra Column")
+    with st.container(border=True):
+
+        st.selectbox("extra column", post.df.columns, key="extra_col",
+                     help="{0} {1}".format("Optional extra columns to display in the filtered DataFrame.",
+                                           "This does not affect plotting."))
+        st.button("Add Extra Column", on_click=add_extra_column)
+
+
+def update_extra_column():
+    """
+        Apply user-selected extra columns to session state config.
+    """
+
+    state = st.session_state
+    config = state.config
+    # align states
+    config.extra_columns = state.extra_columns
+
+
+def add_extra_column():
+    """
+        Allow the user to add a new extra column to session state config.
+    """
+
+    state = st.session_state
+    config = state.config
+
+    if state.extra_col not in state.config.extra_columns:
+        # add extra column to list
+        config.extra_columns.append(state.extra_col)
+        if (len(config.all_columns + config.extra_columns) !=
+            len(set(config.all_columns + config.extra_columns))):
+            st.warning("Column already present in config. Removing from extra columns list.")
+        # re-parse column names
+        config.parse_columns()
+
+
 def rerun_post_processing():
     """
         Run post-processing with the current session state config.