diff --git a/calculation_history.ipynb b/calculation_history.ipynb index a60d745fb..11a1130ff 100644 --- a/calculation_history.ipynb +++ b/calculation_history.ipynb @@ -6,73 +6,11 @@ "metadata": {}, "outputs": [], "source": [ - "import ipywidgets as ipw\n", - "\n", - "how_to = ipw.HTML(\n", - " \"\"\"\n", - " \n", - "
Calculation history
\n", - "
\n", - "

How to use this page

\n", - "

\n", - " This page allows you to view and manage your calculation history. Use the table below to\n", - " see all jobs in the database. You can use the following filters to narrow down your search:\n", - "

\n", - " \n", - "

\n", - " Each row in the table provides links to inspect, delete or download a job. To delete a job, click the \"Delete\"\n", - " link in the respective row. To view detailed information about a job, click the \"PK\" link. To download the\n", - " input/output files of a job, click the \"Download\" link.\n", - "

\n", - "
\n", - " \"\"\"\n", - ")\n", - "\n", - "how_to" + "%%javascript\n", + "IPython.OutputArea.prototype._should_scroll = function(lines) {\n", + " return false;\n", + "}\n", + "document.title='AiiDAlab QE app calculation history'" ] }, { @@ -104,11 +42,66 @@ "metadata": {}, "outputs": [], "source": [ - "from aiidalab_qe.app.utils.search_jobs import QueryInterface\n", + "import ipywidgets as ipw\n", + "from importlib_resources import files\n", + "from jinja2 import Environment\n", + "\n", + "from aiidalab_qe.app.static import templates\n", + "from aiidalab_qe.common.infobox import InfoBox\n", + "\n", + "title = ipw.HTML(\n", + " \"

Calculation history

\", layout=ipw.Layout(margin=\"0 0 15px 0\")\n", + ")\n", + "env = Environment()\n", + "guide_template = (\n", + " files(templates).joinpath(\"calculation_history_guide.jinja\").read_text()\n", + ")\n", + "guide = ipw.HTML(env.from_string(guide_template).render())\n", + "\n", + "\n", + "info_container = InfoBox(layout=ipw.Layout(margin=\"14px 2px 0\"))\n", + "guide_toggle = ipw.ToggleButton(\n", + " layout=ipw.Layout(width=\"150px\"),\n", + " button_style=\"\",\n", + " icon=\"book\",\n", + " value=False,\n", + " description=\"Page guide\",\n", + " tooltip=\"Learn how to use the app\",\n", + " disabled=False,\n", + ")\n", + "\n", + "\n", + "def _on_guide_toggle(change: dict):\n", + " \"\"\"Toggle the guide section.\"\"\"\n", + " if change[\"new\"]:\n", + " info_container.children = [\n", + " guide,\n", + " ]\n", + " info_container.layout.display = \"flex\"\n", + " else:\n", + " info_container.children = []\n", + " info_container.layout.display = \"none\"\n", + "\n", + "\n", + "guide_toggle.observe(\n", + " _on_guide_toggle,\n", + " \"value\",\n", + ")\n", + "\n", + "controls = ipw.VBox(children=[title, guide_toggle, info_container])\n", + "controls" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from aiidalab_qe.app.utils.search_jobs import CalculationHistory\n", "\n", - "calculation_history = QueryInterface()\n", - "calculation_history.setup_table()\n", - "calculation_history.filters_layout" + "calculation_history = CalculationHistory()\n", + "calculation_history.main" ] }, { @@ -117,7 +110,7 @@ "metadata": {}, "outputs": [], "source": [ - "calculation_history.table" + "calculation_history.load_table()" ] } ], diff --git a/setup.cfg b/setup.cfg index c594b09fd..d7716144c 100644 --- a/setup.cfg +++ b/setup.cfg @@ -34,6 +34,8 @@ install_requires = aiida-wannier90-workflows==2.3.0 pymatgen==2024.5.1 anywidget==0.9.13 + table_widget~=0.0.2 + python_requires = >=3.9 [options.packages.find] diff --git a/src/aiidalab_qe/app/static/templates/calculation_history_guide.jinja b/src/aiidalab_qe/app/static/templates/calculation_history_guide.jinja new file mode 100644 index 000000000..b09475734 --- /dev/null +++ b/src/aiidalab_qe/app/static/templates/calculation_history_guide.jinja @@ -0,0 +1,28 @@ +
+

How to use this page

+

+ This page allows you to view, filter, sort, and manage your calculation history. +

+

Filters and search

+ +

Table actions

+ +

Display options

+ +
diff --git a/src/aiidalab_qe/app/utils/search_jobs.py b/src/aiidalab_qe/app/utils/search_jobs.py index 1b899c50d..851c6994d 100644 --- a/src/aiidalab_qe/app/utils/search_jobs.py +++ b/src/aiidalab_qe/app/utils/search_jobs.py @@ -1,10 +1,12 @@ +from datetime import datetime, timezone + import ipywidgets as ipw -import pandas as pd -from IPython.display import display +from table_widget import TableWidget -from aiida.orm import QueryBuilder +from aiida.orm import QueryBuilder, load_node +from aiidalab_qe.common.widgets import LoadingWidget -state_icons = { +STATE_ICONS = { "running": "⏳", "finished": "✅", "excepted": "⚠️", @@ -12,16 +14,82 @@ } -class QueryInterface: +def determine_state_icon(row): + """Attach an icon to the displayed job state.""" + state = row["state"].lower() + if state == "finished" and row.get("exit_status", 0) != 0: + return f"Finished{STATE_ICONS['excepted']}" + return f"{state.capitalize()}{STATE_ICONS.get(state, '')}" + + +COLUMNS = { + "id": {"headerName": "ID 🔗", "dataType": "link", "editable": False}, + "creation_time_absolute": { + "headerName": "Creation time ⏰ (absolute)", + "type": "date", + "width": 100, + "editable": False, + }, + "creation_time_relative": { + "headerName": "Creation time ⏰ (relative)", + "width": 100, + "editable": False, + }, + "structure": {"headerName": "Structure", "editable": False}, + "state": {"headerName": "State 🟢", "editable": False}, + "status": {"headerName": "Status", "editable": False, "hide": True}, + "exit_status": { + "headerName": "Exit status", + "type": "number", + "editable": False, + "hide": True, + }, + "exit_message": {"headerName": "Exit message", "editable": False}, + "label": {"headerName": "Label", "width": 300, "editable": True}, + "description": { + "headerName": "Description", + "width": 300, + "editable": True, + "hide": True, + }, + "relax_type": {"headerName": "Relax type", "editable": False, "hide": True}, + "delete": {"headerName": "Delete", "dataType": "link", "editable": False}, + "download": {"headerName": "Download", "dataType": "link", "editable": False}, + "uuid": {"headerName": "UUID", "editable": False, "hide": True}, + "properties": {"headerName": "Properties", "editable": False, "hide": True}, +} + + +class CalculationHistory: def __init__(self): - pass + self.main = ipw.VBox(children=[LoadingWidget("Loading the table...")]) + + self.table = TableWidget(style={"margin-top": "20px"}) + + def on_row_update(change): + # When the user updates 'label' or 'description' in the table, + # reflect these changes in the corresponding AiiDA node. + node = load_node(change["new"]["uuid"]) + node.label = change["new"]["label"] + node.description = change["new"]["description"] + + self.table.observe(on_row_update, "updatedRow") - def setup_table(self): - self.df = self.load_data() - self.table = ipw.HTML() + # This will hold the raw data (list of dicts) from the database + self.data = [] + # This will hold the currently displayed data (filtered, with toggles applied, etc.) + self.display_data = [] + + def load_table(self): + """Populate the table after initialization.""" + self.data = self.load_data() self.setup_widgets() def load_data(self): + """Fetch the QeAppWorkChain results using the QueryBuilder and + return a list of dictionaries with all required columns. + + """ from aiidalab_qe.workflows import QeAppWorkChain projections = [ @@ -30,107 +98,115 @@ def load_data(self): "extras.structure", "ctime", "attributes.process_state", + "attributes.process_status", + "attributes.exit_status", + "attributes.exit_message", "label", "description", "extras.workchain.relax_type", "extras.workchain.properties", ] - headers = [ - "PK", - "UUID", - "Structure", - "ctime", - "State", - "Label", - "Description", - "Relax_type", - "Properties", - ] qb = QueryBuilder() qb.append(QeAppWorkChain, project=projections, tag="process") qb.order_by({"process": {"ctime": "desc"}}) results = qb.all() - df = pd.DataFrame(results, columns=headers) - # Check if DataFrame is not empty - if not df.empty: - df["Creation time"] = df["ctime"].apply( - lambda x: x.strftime("%Y-%m-%d %H:%M:%S") - ) - df["Delete"] = df["PK"].apply( - lambda pk: f'Delete' - ) - df["Download"] = df["PK"].apply( - lambda pk: f'Download' + data = [] + if not results: + return data + + now = datetime.now(timezone.utc) + + for row in results: + ( + pk, + uuid, + structure, + creation_time, + state, + status, + exit_status, + exit_message, + label, + description, + relax_type, + properties, + ) = row + + creation_time_str = ( + creation_time.strftime("%Y-%m-%d %H:%M:%S") if creation_time else "" ) - # add a link to the pk so that the user can inspect the calculation - df["PK_with_link"] = df["PK"].apply( - lambda pk: f'{pk}' + if creation_time: + days_ago = (now - creation_time).days + creation_time_rel = f"{days_ago}D ago" + else: + creation_time_rel = "N/A" + + # Transform "waiting" to "running" for readbility + if state == "waiting": + state = "running" + + # Prepare link-based values + pk_with_link = f'{pk}' + uuid_with_link = ( + f'{uuid[:8]}' ) - # Store initial part of the UUID - df["UUID_with_link"] = df.apply( - lambda row: f'{row["UUID"][:8]}', - axis=1, + delete_link = f'Delete' + download_link = ( + f'Download' ) - # replace all "waiting" states with "running" - df["State"] = df["State"].apply( - lambda x: "running" if x == "waiting" else x + + # Make sure properties is a list (avoid None) + properties = properties if properties is not None else [] + + data.append( + { + "pk_with_link": pk_with_link, + "uuid_with_link": uuid_with_link, + "creation_time_absolute": creation_time_str, + "creation_time_relative": creation_time_rel, + "structure": structure, + "state": state, + "status": status, + "exit_status": exit_status, + "exit_message": exit_message, + "label": label, + "description": description, + "relax_type": relax_type, + "uuid": uuid, + "delete": delete_link, + "download": download_link, + "properties": properties, + "creation_time": creation_time, + } ) - else: - # Initialize empty columns for an empty DataFrame - df["Creation time"] = pd.Series(dtype="str") - df["Delete"] = pd.Series(dtype="str") - return df[ - [ - "PK_with_link", - "UUID_with_link", - "Creation time", - "Structure", - "State", - "Label", - "Description", - "Relax_type", - "Delete", - "Download", - "Properties", - "ctime", - ] - ] + + return data def setup_widgets(self): - self.css_style = """ - - """ - - unique_properties = set(self.df["Properties"].explode().dropna()) - unique_properties.discard(None) + """Create widgets for filtering, toggles for display, etc.""" + # Gather unique properties + all_properties = set() + for row in self.data: + for prop in row["properties"]: + if prop is not None: + all_properties.add(prop) + + # Build a set of checkboxes for properties property_checkboxes = [ ipw.Checkbox( value=False, description=prop, - Layout=ipw.Layout(description_width="initial"), indent=False, + layout=ipw.Layout(description_width="initial"), ) - for prop in unique_properties + for prop in sorted(all_properties) ] - self.properties_box = ipw.HBox( - children=property_checkboxes, description="Properties:" - ) - self.properties_filter_description = ipw.HTML( - "

Properties filter: Select one or more properties to narrow the results. Only calculations that include all the selected properties will be displayed. Leave all checkboxes unselected to include calculations regardless of their properties.

" - ) - # Replace 'None' in 'Properties' with an empty list - self.df["Properties"] = self.df["Properties"].apply( - lambda x: [] if x is None else x - ) + + self.properties_box = ipw.HBox(property_checkboxes) + self.properties_filter_description = ipw.HTML("Filter by properties:") + self.job_state_dropdown = ipw.Dropdown( options={ "Any": "", @@ -142,166 +218,161 @@ def setup_widgets(self): value="", # Default value corresponding to "Any" description="Job state:", ) - self.label_search_field = ipw.Text( - value="", - placeholder="Enter a keyword", - description="", - disabled=False, - style={"description_width": "initial"}, - ) - self.label_search_description = ipw.HTML( - "

Search label: Enter a keyword to search in both the Label and Description fields. Matches will include any calculations where the keyword is found in either field.

" - ) - self.toggle_description_checkbox = ipw.Checkbox( - value=False, # Show the Description column by default - description="Show description", - indent=False, - ) - self.toggle_description_checkbox.observe( - self.update_table_visibility, names="value" - ) + self.toggle_time_format = ipw.ToggleButtons( options=["Absolute", "Relative"], - value="Absolute", # Default to Absolute time + value="Absolute", # Default to showing Absolute time description="Time format:", ) self.toggle_time_format.observe(self.update_table_visibility, names="value") + self.toggle_id_format = ipw.ToggleButtons( - options=["PK", "UUID"], - value="PK", # Default to PK + options=["pk", "uuid"], + value="pk", description="ID format:", ) self.toggle_id_format.observe(self.update_table_visibility, names="value") + # Date pickers for range-based filtering self.time_start = ipw.DatePicker(description="Start time:") self.time_end = ipw.DatePicker(description="End time:") self.time_box = ipw.HBox([self.time_start, self.time_end]) - # self.apply_filters_btn = ipw.Button(description='Apply Filters') - # self.apply_filters_btn.on_click(self.apply_filters) + + # Connect checkboxes and dropdowns to the filter logic for cb in property_checkboxes: cb.observe(self.apply_filters, names="value") self.time_start.observe(self.apply_filters, names="value") self.time_end.observe(self.apply_filters, names="value") self.job_state_dropdown.observe(self.apply_filters, names="value") - self.label_search_field.observe(self.apply_filters, names="value") - self.filters_layout = ipw.VBox( - [ - ipw.HTML("

Search & filter calculations:

"), + display_options = ipw.VBox( + children=[ + ipw.VBox(children=[self.toggle_time_format, self.toggle_id_format]), + ], + layout=ipw.Layout( + border="1px solid lightgray", + padding="0.5em", + ), + ) + + filters = ipw.VBox( + children=[ ipw.VBox( - [ + children=[ self.job_state_dropdown, self.time_box, - ipw.HBox( - [ - self.label_search_description, - self.label_search_field, - ] - ), ipw.VBox( - [self.properties_filter_description, self.properties_box] + children=[ + self.properties_filter_description, + self.properties_box, + ], + layout=ipw.Layout( + border="1px solid lightgray", + padding="0.5em", + margin="5px", + ), ), - # self.apply_filters_btn, - ] - ), - ipw.HTML("

Display options:

"), - ipw.VBox( - [ - self.toggle_description_checkbox, - self.toggle_time_format, - self.toggle_id_format, ] ), - ] + ], + layout=ipw.Layout( + border="1px solid lightgray", + padding="0.5em", + ), ) - self.get_table_value(self.df) - - def get_table_value(self, display_df): - if display_df.empty: - self.table.value = "

No results found

" - return - # Adjust the Creation time column based on the toggle state - if self.toggle_time_format.value == "Relative": - now = pd.Timestamp.now(tz="UTC") - display_df["Creation time"] = display_df["ctime"].apply( - lambda x: f"{(now - x).days}D ago" if pd.notnull(x) else "N/A" - ) - else: - display_df["Creation time"] = display_df["ctime"].apply( - lambda x: x.strftime("%Y-%m-%d %H:%M:%S") if pd.notnull(x) else "N/A" - ) - # Conditionally drop the Description column based on the checkbox state - if not self.toggle_description_checkbox.value: - display_df = display_df.drop(columns=["Description"]) - - # Adjust the ID column based on the toggle state - if self.toggle_id_format.value == "PK": - display_df = display_df.rename(columns={"PK_with_link": "ID"}).drop( - columns=["UUID_with_link"] - ) - else: - display_df = display_df.rename(columns={"UUID_with_link": "ID"}).drop( - columns=["PK_with_link"] - ) - # - display_df["State"] = display_df["State"].apply( - lambda x: f"{x.capitalize()}{state_icons.get(x.lower())}" - ) - display_df.rename( - columns={ - "State": "State 🟢", - "Creation time": "Creation time ⏰", - "ID": "ID 🔗", - "Relax_type": "Relax type", - }, - inplace=True, + self.main.children = [ + ipw.HTML("

Display options:

"), + display_options, + ipw.HTML("

Filters:

"), + filters, + self.table, + ] + + self.update_table_value(self.data) + + def update_table_value(self, data_list): + """Prepare the data to be shown (adding or hiding columns, etc.), + and load it into `self.table`. + """ + # Adjust which creation_time columns to hide + COLUMNS["creation_time_relative"]["hide"] = ( + self.toggle_time_format.value != "Relative" ) - display_df = display_df.drop(columns=["Properties", "ctime"]) - self.table.value = self.css_style + display_df.to_html( - classes="df", escape=False, index=False + COLUMNS["creation_time_absolute"]["hide"] = ( + self.toggle_time_format.value != "Absolute" ) + # Build a new list that has an 'id' column, etc. + display_data = [] + for row in data_list: + row_copy = dict(row) + # Switch the 'id' column depending on pk vs uuid toggle + if self.toggle_id_format.value == "pk": + row_copy["id"] = row_copy["pk_with_link"] + else: + row_copy["id"] = row_copy["uuid_with_link"] + + # Overwrite "state" with icon-based representation + row_copy["state"] = determine_state_icon(row_copy) + display_data.append(row_copy) + + # Figure out which columns to show the table + columns = [] + for key in COLUMNS.keys(): + col_spec = dict(COLUMNS[key]) + col_spec["field"] = key + columns.append(col_spec) + + self.table.from_data(display_data, columns=columns) + def apply_filters(self, _): + """Filter the raw data based on job state, selected properties, + and date range. Then update the table display. + """ + filtered = [] + selected_properties = [ cb.description for cb in self.properties_box.children if cb.value ] - filtered_df = self.df.copy() - filtered_df = filtered_df[ - filtered_df["State"].str.contains(self.job_state_dropdown.value) - ] - if self.label_search_field.value: - filtered_df = filtered_df[ - filtered_df["Label"].str.contains( - self.label_search_field.value, case=False, na=False - ) - | filtered_df["Description"].str.contains( - self.label_search_field.value, case=False, na=False - ) - ] - if selected_properties: - filtered_df = filtered_df[ - filtered_df["Properties"].apply( - lambda x: all(item in x for item in selected_properties) - ) - ] - if self.time_start.value and self.time_end.value: - start_time = pd.to_datetime(self.time_start.value).normalize() - end_time = pd.to_datetime(self.time_end.value).normalize() + pd.Timedelta( - days=1, milliseconds=-1 - ) - start_time = start_time.tz_localize("UTC") - end_time = end_time.tz_localize("UTC") - filtered_df = filtered_df[ - (filtered_df["ctime"] >= start_time) - & (filtered_df["ctime"] <= end_time) - ] - self.get_table_value(filtered_df) + + # Convert DatePicker values (which are dates) into datetimes with UTC + start_time = None + end_time = None + if self.time_start.value is not None: + start_time = datetime.combine(self.time_start.value, datetime.min.time()) + start_time = start_time.replace(tzinfo=timezone.utc) + + if self.time_end.value is not None: + end_time = datetime.combine(self.time_end.value, datetime.max.time()) + end_time = end_time.replace(tzinfo=timezone.utc) + + # State filter (empty string means "Any") + desired_state_substring = self.job_state_dropdown.value + + for row in self.data: + if desired_state_substring: + if desired_state_substring not in row["state"].lower(): + continue + + row_props = row["properties"] + if selected_properties: + # Must have all selected properties in row_props + if not all(prop in row_props for prop in selected_properties): + continue + + ctime = row.get("creation_time", None) + if ctime is not None: + if start_time and ctime < start_time: + continue + if end_time and ctime > end_time: + continue + + filtered.append(row) + + self.update_table_value(filtered) def update_table_visibility(self, _): - # Reapply filters to refresh the table visibility when the checkbox changes + """Called when toggles for time format or ID format change.""" + # simply re-apply filters (which triggers a re-draw). self.apply_filters(None) - - def display(self): - display(self.filters_layout) - display(self.table) diff --git a/tests/test_calculation_history.py b/tests/test_calculation_history.py new file mode 100644 index 000000000..10a7c8196 --- /dev/null +++ b/tests/test_calculation_history.py @@ -0,0 +1,9 @@ +def test_calculation_history(sssp, generate_qeapp_workchain): + from aiidalab_qe.app.utils.search_jobs import CalculationHistory + + workchain = generate_qeapp_workchain() + workchain.node.seal() + + calculation_history = CalculationHistory() + calculation_history.load_table() + assert len(calculation_history.table.data) >= 1