Merge pull request #384 from lsst/tickets/DM-41321

DM-41321: Add try-except block to manifest checker so that failure to get butler logs does not make error reporting impossible

python/lsst/pipe/base/execution_reports.py

@@ -230,14 +230,14 @@ def inspect_quantum(
             else:
                 dataset_type_report.handle_produced_dataset(output_ref, status_graph)
 
-    def to_summary_dict(self, butler: Butler, logs: bool = True) -> dict[str, Any]:
+    def to_summary_dict(self, butler: Butler, do_store_logs: bool = True) -> dict[str, Any]:
         """Summarize the results of the TaskExecutionReport in a dictionary.
 
         Parameters
         ----------
         butler : `lsst.daf.butler.Butler`
             The Butler used for this report.
-        logs : `bool`
+        do_store_logs : `bool`
             Store the logs in the summary dictionary.
 
         Returns
@@ -257,11 +257,13 @@ def to_summary_dict(self, butler: Butler, logs: bool = True) -> dict[str, Any]:
         failed_quanta = {}
         for node_id, log_ref in self.failed.items():
             quantum_info: dict[str, Any] = {"data_id": log_ref.dataId.byName()}
-            if logs:
+            if do_store_logs:
                 try:
                     log = butler.get(log_ref)
                 except LookupError:
                     quantum_info["error"] = []
+                except FileNotFoundError:
+                    quantum_info["error"] = None
                 else:
                     quantum_info["error"] = [
                         record.message for record in log if record.levelno >= logging.ERROR
@@ -305,15 +307,15 @@ class QuantumGraphExecutionReport:
     """A dictionary of TaskExecutionReports by task label (`dict`).
     """
 
-    def to_summary_dict(self, butler: Butler, logs: bool = True) -> dict[str, Any]:
+    def to_summary_dict(self, butler: Butler, do_store_logs: bool = True) -> dict[str, Any]:
         """Summarize the results of the `QuantumGraphExecutionReport` in a
         dictionary.
 
         Parameters
         ----------
         butler : `lsst.daf.butler.Butler`
             The Butler used for this report.
-        logs : `bool`
+        do_store_logs : `bool`
             Store the logs in the summary dictionary.
 
         Returns
@@ -322,9 +324,12 @@ def to_summary_dict(self, butler: Butler, logs: bool = True) -> dict[str, Any]:
             A dictionary containing a summary of a `TaskExecutionReport` for
             each task in the quantum graph.
         """
-        return {task: report.to_summary_dict(butler, logs=logs) for task, report in self.tasks.items()}
+        return {
+            task: report.to_summary_dict(butler, do_store_logs=do_store_logs)
+            for task, report in self.tasks.items()
+        }
 
-    def write_summary_yaml(self, butler: Butler, filename: str, logs: bool = True) -> None:
+    def write_summary_yaml(self, butler: Butler, filename: str, do_store_logs: bool = True) -> None:
         """Take the dictionary from
         `QuantumGraphExecutionReport.to_summary_dict` and store its contents in
         a yaml file.
@@ -335,11 +340,11 @@ def write_summary_yaml(self, butler: Butler, filename: str, logs: bool = True) -
             The Butler used for this report.
         filename : `str`
             The name to be used for the summary yaml file.
-        logs : `bool`
+        do_store_logs : `bool`
             Store the logs in the summary dictionary.
         """
         with open(filename, "w") as stream:
-            yaml.safe_dump(self.to_summary_dict(butler, logs=logs), stream)
+            yaml.safe_dump(self.to_summary_dict(butler, do_store_logs=do_store_logs), stream)
 
     @classmethod
     def make_reports(

tests/test_execution_reports.py

@@ -47,7 +47,7 @@ def test_make_reports(self) -> None:
             # make a simple qgraph to make an execution report on
             butler, qgraph = simpleQGraph.makeSimpleQGraph(root=root)
             report = QuantumGraphExecutionReport.make_reports(butler, qgraph)
-            dict_of_expected_failures = report.to_summary_dict(butler, logs=False)
+            dict_of_expected_failures = report.to_summary_dict(butler, do_store_logs=False)
             self.assertIsNotNone(dict_of_expected_failures["task0"]["failed_quanta"])
             self.assertEqual(
                 dict_of_expected_failures["task1"]["outputs"]["add_dataset2"]["missing_upstream_failed"], 1