diff --git a/doc/changes/DM-41606.feature.md b/doc/changes/DM-41606.feature.md new file mode 100644 index 00000000..d1b1b9f8 --- /dev/null +++ b/doc/changes/DM-41606.feature.md @@ -0,0 +1,7 @@ +Make option to output `pipetask report` information to the command-line using +astropy tables and set to default. +Unpack a more human-readable dictionary from +`lsst.pipe.base.QuantumGraphExecutionReports.to_summary_dict` and print summary +tables of quanta and datasets to the command-line. Save error messages and +associated data ids to a yaml file in the working directory, or optionally print +them to screen as well. diff --git a/python/lsst/ctrl/mpexec/cli/cmd/commands.py b/python/lsst/ctrl/mpexec/cli/cmd/commands.py index 75547294..cedd12de 100644 --- a/python/lsst/ctrl/mpexec/cli/cmd/commands.py +++ b/python/lsst/ctrl/mpexec/cli/cmd/commands.py @@ -329,9 +329,11 @@ def update_graph_run( @click.command(cls=PipetaskCommand) @repo_argument() @ctrlMpExecOpts.qgraph_argument() -@click.option("--output-yaml", default='', help="Summarize report in a yaml file (pass its name here).") +@click.option("--full-output-filename", default='', help="Summarize report in a yaml file") @click.option("--logs/--no-logs", default=True, help="Get butler log datasets for extra information.") -def report(repo: str, qgraph: str, output_yaml: str='', logs: bool = True) -> None: +@click.option("--show-errors", default=False, help="Pretty-print a dict of errors from failed quanta.") +def report(repo: str, qgraph: str, full_output_filename: str = '', logs: bool = True, + show_errors: bool = False) -> None: """Write a yaml file summarizing the produced and missing expected datasets in a quantum graph. @@ -339,4 +341,4 @@ def report(repo: str, qgraph: str, output_yaml: str='', logs: bool = True) -> No QGRAPH is the URL to a serialized Quantum Graph file. """ - script.report(repo, qgraph, output_yaml, logs) + script.report(repo, qgraph, full_output_filename, logs) diff --git a/python/lsst/ctrl/mpexec/cli/script/report.py b/python/lsst/ctrl/mpexec/cli/script/report.py index 3c2c11c9..4fc973dd 100644 --- a/python/lsst/ctrl/mpexec/cli/script/report.py +++ b/python/lsst/ctrl/mpexec/cli/script/report.py @@ -25,38 +25,51 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . import pprint -from astropy.table import Table +import yaml +from astropy.table import Table from lsst.daf.butler import Butler from lsst.pipe.base import QuantumGraph from lsst.pipe.base.execution_reports import QuantumGraphExecutionReport -def report(butler_config: str, qgraph_uri: str, output_yaml: str | None, logs: bool = True) -> None: - """Write a yaml file summarizing the produced and missing expected datasets - in a quantum graph. +def report(butler_config: str, qgraph_uri: str, + full_output_filename: str | None, logs: bool = True, + show_errors: bool = False) -> None: + """Summarize the produced and missing expected dataset in a quantum graph. Parameters ---------- - butler_config : `str` - The Butler used for this report. This should match the Butler used - for the run associated with the executed quantum graph. - qgraph_uri : `str` - The uri of the location of said quantum graph. - output_yaml : `str` - The name to be used for the summary yaml file. - logs : `bool` - Get butler log datasets for extra information. + butler_config : `str` + The Butler used for this report. This should match the Butler used + for the run associated with the executed quantum graph. + qgraph_uri : `str` + The uri of the location of said quantum graph. + full_output_filename : `str` + Output the full summary report to a yaml file (named herein). + Each data id and error message is keyed to a quantum graph node id. + A convenient output format for error-matching and cataloguing tools + such as the ones in the Campaign Management database. If this is + not included, quanta and dataset information will be printed to the + command-line instead. + logs : `bool` + Get butler log datasets for extra information (error messages). + show_errors : `bool` + If no output yaml is provided, print error messages to the + command-line along with the report. By default, these messages and + their associated data ids are stored in a yaml file with format + `{run timestamp}_err.yaml` in the working directory instead. See Also -------- - lsst.pipe.base.QuantumGraphExecutionReport.make_reports : Making reports. - lsst.pipe.base.QuantumGraphExecutionReport.write_summary_yaml : Summaries. + lsst.pipe.base.QuantumGraphExecutionReport.make_reports + lsst.pipe.base.QuantumGraphExecutionReport.write_summary_yaml + lsst.pipe.base.TaskExecutionReport.to_summary_dict """ butler = Butler.from_config(butler_config, writeable=False) qgraph = QuantumGraph.loadUri(qgraph_uri) report = QuantumGraphExecutionReport.make_reports(butler, qgraph) - if not output_yaml: + if not full_output_filename: # this is the option to print to the command-line summary_dict = report.to_summary_dict(butler, logs, human_readable=True) dataset_table_rows = [] @@ -72,14 +85,22 @@ def report(butler_config: str, qgraph_uri: str, output_yaml: str | None, logs: b "Blocked Quanta": summary_dict[task]["n_quanta_blocked"]}) if 'errors' in summary_dict[task].keys(): - error_summary.append(summary_dict[task]["errors"]) + error_summary.append({task: summary_dict[task]["errors"]}) quanta = Table(quanta_summary) datasets = Table(dataset_table_rows) datasets.add_column(data_products, index=0, name="DatasetType") quanta.pprint_all() print("\n") - pprint.pprint(error_summary) - print("\n") + if show_errors: + pprint.pprint(error_summary) + print("\n") + else: + assert qgraph.metadata is not None, "Saved QGs always have metadata." + collection = qgraph.metadata["output_run"] + collection = str(collection) + run_name = collection.split("/")[-1] + with open(f"{run_name}_err.yaml", "w") as stream: + yaml.safe_dump(error_summary, stream) datasets.pprint_all() else: - report.write_summary_yaml(butler, output_yaml, do_store_logs=logs) + report.write_summary_yaml(butler, full_output_filename, do_store_logs=logs) diff --git a/tests/test_cliCmdReport.py b/tests/test_cliCmdReport.py index ab26dd21..f05b62c1 100644 --- a/tests/test_cliCmdReport.py +++ b/tests/test_cliCmdReport.py @@ -71,7 +71,7 @@ def test_report(self): result = self.runner.invoke( pipetask_cli, - ["report", self.root, graph_uri, test_filename, "--no-logs"], + ["report", self.root, graph_uri, "--full-output-filename", test_filename, "--no-logs"], input="no", )