Save error output in a yaml file, adding an option to print it to scr…

…een instead.

doc/changes/DM-41606.feature.md

@@ -0,0 +1,7 @@
+Make option to output `pipetask report` information to the command-line using
+astropy tables and set to default.
+Unpack a more human-readable dictionary from
+`lsst.pipe.base.QuantumGraphExecutionReports.to_summary_dict` and print summary
+tables of quanta and datasets to the command-line. Save error messages and
+associated data ids to a yaml file in the working directory, or optionally print
+them to screen as well.

python/lsst/ctrl/mpexec/cli/cmd/commands.py

@@ -329,14 +329,16 @@ def update_graph_run(
 @click.command(cls=PipetaskCommand)
 @repo_argument()
 @ctrlMpExecOpts.qgraph_argument()
-@click.option("--output-yaml", default='', help="Summarize report in a yaml file (pass its name here).")
+@click.option("--full-output-filename", default='', help="Summarize report in a yaml file")
 @click.option("--logs/--no-logs", default=True, help="Get butler log datasets for extra information.")
-def report(repo: str, qgraph: str, output_yaml: str='', logs: bool = True) -> None:
+@click.option("--show-errors", default=False, help="Pretty-print a dict of errors from failed quanta.")
+def report(repo: str, qgraph: str, full_output_filename: str = '', logs: bool = True,
+           show_errors: bool = False) -> None:
     """Write a yaml file summarizing the produced and missing expected datasets
     in a quantum graph.
 
     REPO is the location of the butler/registry config file.
 
     QGRAPH is the URL to a serialized Quantum Graph file.
     """
-    script.report(repo, qgraph, output_yaml, logs)
+    script.report(repo, qgraph, full_output_filename, logs)

python/lsst/ctrl/mpexec/cli/script/report.py

@@ -25,38 +25,51 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 import pprint
-from astropy.table import Table
 
+import yaml
+from astropy.table import Table
 from lsst.daf.butler import Butler
 from lsst.pipe.base import QuantumGraph
 from lsst.pipe.base.execution_reports import QuantumGraphExecutionReport
 
 
-def report(butler_config: str, qgraph_uri: str, output_yaml: str | None, logs: bool = True) -> None:
-    """Write a yaml file summarizing the produced and missing expected datasets
-    in a quantum graph.
+def report(butler_config: str, qgraph_uri: str,
+           full_output_filename: str | None, logs: bool = True,
+           show_errors: bool = False) -> None:
+    """Summarize the produced and missing expected dataset in a quantum graph.
 
     Parameters
     ----------
-    butler_config : `str`
-        The Butler used for this report. This should match the Butler used
-        for the run associated with the executed quantum graph.
-    qgraph_uri : `str`
-        The uri of the location of said quantum graph.
-    output_yaml : `str`
-        The name to be used for the summary yaml file.
-    logs : `bool`
-        Get butler log datasets for extra information.
+        butler_config : `str`
+            The Butler used for this report. This should match the Butler used
+            for the run associated with the executed quantum graph.
+        qgraph_uri : `str`
+            The uri of the location of said quantum graph.
+        full_output_filename : `str`
+            Output the full summary report to a yaml file (named herein).
+            Each data id and error message is keyed to a quantum graph node id.
+            A convenient output format for error-matching and cataloguing tools
+            such as the ones in the Campaign Management database. If this is
+            not included, quanta and dataset information will be printed to the
+            command-line instead.
+        logs : `bool`
+            Get butler log datasets for extra information (error messages).
+        show_errors : `bool`
+            If no output yaml is provided, print error messages to the
+            command-line along with the report. By default, these messages and
+            their associated data ids are stored in a yaml file with format
+            `{run timestamp}_err.yaml` in the working directory instead.
 
     See Also
     --------
-    lsst.pipe.base.QuantumGraphExecutionReport.make_reports : Making reports.
-    lsst.pipe.base.QuantumGraphExecutionReport.write_summary_yaml : Summaries.
+        lsst.pipe.base.QuantumGraphExecutionReport.make_reports
+        lsst.pipe.base.QuantumGraphExecutionReport.write_summary_yaml
+        lsst.pipe.base.TaskExecutionReport.to_summary_dict
     """
     butler = Butler.from_config(butler_config, writeable=False)
     qgraph = QuantumGraph.loadUri(qgraph_uri)
     report = QuantumGraphExecutionReport.make_reports(butler, qgraph)
-    if not output_yaml:
+    if not full_output_filename:
         # this is the option to print to the command-line
         summary_dict = report.to_summary_dict(butler, logs, human_readable=True)
         dataset_table_rows = []
@@ -72,14 +85,22 @@ def report(butler_config: str, qgraph_uri: str, output_yaml: str | None, logs: b
                                    "Blocked Quanta": summary_dict[task]["n_quanta_blocked"]})
 
             if 'errors' in summary_dict[task].keys():
-                error_summary.append(summary_dict[task]["errors"])
+                error_summary.append({task: summary_dict[task]["errors"]})
         quanta = Table(quanta_summary)
         datasets = Table(dataset_table_rows)
         datasets.add_column(data_products, index=0, name="DatasetType")
         quanta.pprint_all()
         print("\n")
-        pprint.pprint(error_summary)
-        print("\n")
+        if show_errors:
+            pprint.pprint(error_summary)
+            print("\n")
+        else:
+            assert qgraph.metadata is not None, "Saved QGs always have metadata."
+            collection = qgraph.metadata["output_run"]
+            collection = str(collection)
+            run_name = collection.split("/")[-1]
+            with open(f"{run_name}_err.yaml", "w") as stream:
+                yaml.safe_dump(error_summary, stream)
         datasets.pprint_all()
     else:
-        report.write_summary_yaml(butler, output_yaml, do_store_logs=logs)
+        report.write_summary_yaml(butler, full_output_filename, do_store_logs=logs)

tests/test_cliCmdReport.py

@@ -71,7 +71,7 @@ def test_report(self):
 
         result = self.runner.invoke(
             pipetask_cli,
-            ["report", self.root, graph_uri, test_filename, "--no-logs"],
+            ["report", self.root, graph_uri, "--full-output-filename", test_filename, "--no-logs"],
             input="no",
         )