diff --git a/github_pages/make_site_content.sh b/github_pages/make_site_content.sh index bbb1f1f8c2..566a7462a9 100755 --- a/github_pages/make_site_content.sh +++ b/github_pages/make_site_content.sh @@ -11,6 +11,4 @@ mkdir ./public cp -r ./monitoring/github_pages/static/* ./public mkdir -p ./public/artifacts/uss_qualifier/reports -cp -r ./artifacts/uss_qualifier/output/sequence_uspace ./public/artifacts/uss_qualifier/reports/sequence_uspace -cp -r ./artifacts/uss_qualifier/output/tested_requirements_uspace ./public/artifacts/uss_qualifier/reports/tested_requirements_uspace -cp -r ./artifacts/uss_qualifier/output/capabilities_uspace.html ./public/artifacts/uss_qualifier/reports/capabilities_uspace.html +cp -r ./artifacts/uss_qualifier/output ./public/artifacts/uss_qualifier/reports diff --git a/github_pages/static/index.md b/github_pages/static/index.md index 036b0aa6c1..02cd5b3ce0 100644 --- a/github_pages/static/index.md +++ b/github_pages/static/index.md @@ -4,10 +4,32 @@ This site contains content automatically generated by actions in the [InterUSS]( ## uss_qualifier [reports](https://github.com/interuss/monitoring/tree/main/monitoring/uss_qualifier/reports) -These reports were generated during continuous integration for the most recent PR merged to the main branch. +These reports were generated during continuous integration for the most recent PR merged to the main branch. The test configurations producing this output are in [monitoring/uss_qualifier/configurations/dev](https://github.com/interuss/monitoring/tree/main/monitoring/uss_qualifier/configurations/dev). -### [U-space developer](https://github.com/interuss/monitoring/blob/main/monitoring/uss_qualifier/configurations/dev/uspace.yaml) [test configuration](https://github.com/interuss/monitoring/tree/main/monitoring/uss_qualifier/configurations) +### [U-space test configuration](https://github.com/interuss/monitoring/blob/main/monitoring/uss_qualifier/configurations/dev/uspace.yaml) -* [Sequence view](./artifacts/uss_qualifier/reports/sequence_uspace) -* [Tested requirements](./artifacts/uss_qualifier/reports/tested_requirements_uspace) -* [Demonstrated capabilities](./artifacts/uss_qualifier/reports/capabilities_uspace.html) +* [Sequence view](./artifacts/uss_qualifier/reports/uspace/sequence) +* [Tested requirements](./artifacts/uss_qualifier/reports/uspace/requirements) +* [Demonstrated capabilities](./artifacts/uss_qualifier/reports/uspace/capabilities.html) +* [Raw report](./artifacts/uss_qualifier/reports/uspace/report.json) (large) + +### [No-op test configuration](https://github.com/interuss/monitoring/blob/main/monitoring/uss_qualifier/configurations/dev/noop.yaml) + +* [Raw report](./artifacts/uss_qualifier/reports/noop/report.json) (indented to be human-readable) +* [Interactive report](./artifacts/uss_qualifier/reports/noop/report.html) +* [Sequence view](./artifacts/uss_qualifier/reports/noop/sequence) + +### [ASTM F3548-21 test configuration](https://github.com/interuss/monitoring/blob/main/monitoring/uss_qualifier/configurations/dev/f3548_self_contained.yaml) + +* [Sequence view](./artifacts/uss_qualifier/reports/f3548/sequence) +* [Tested requirements](./artifacts/uss_qualifier/reports/f3548/requirements) + +### [ASTM F3411-22a test configuration](https://github.com/interuss/monitoring/blob/main/monitoring/uss_qualifier/configurations/dev/netrid_v22a.yaml) + +* [Sequence view](./artifacts/uss_qualifier/reports/netrid_v22a/sequence) +* [Tested requirements](./artifacts/uss_qualifier/reports/netrid_v22a/requirements) + +### [DSS integration test configuration](https://github.com/interuss/monitoring/blob/main/monitoring/uss_qualifier/configurations/dev/dss_probing.yaml) + +* [Sequence view](./artifacts/uss_qualifier/reports/dss_probing/sequence) +* [Tested requirements](./artifacts/uss_qualifier/reports/dss_probing/requirements) diff --git a/monitoring/uss_qualifier/configurations/configuration.py b/monitoring/uss_qualifier/configurations/configuration.py index e3c679323e..cd6526cd2f 100644 --- a/monitoring/uss_qualifier/configurations/configuration.py +++ b/monitoring/uss_qualifier/configurations/configuration.py @@ -27,18 +27,13 @@ class TestConfiguration(ImplicitDict): """Declarations for resources used by the test suite""" -class TestedRolesConfiguration(ImplicitDict): - report_path: str - """Path of folder to write HTML files containing a fulfilled-requirements-based view of the test report""" - - TestedRequirementsCollectionIdentifier = str """Identifier for a requirements collection, local to a TestedRequirementsConfiguration artifact configuration.""" class TestedRequirementsConfiguration(ImplicitDict): - output_path: str - """Path of a folder into which report HTML files should be written""" + report_name: str + """Name of subfolder in output path to contain the rendered templated report""" requirement_collections: Optional[ Dict[TestedRequirementsCollectionIdentifier, RequirementCollection] @@ -52,13 +47,13 @@ class TestedRequirementsConfiguration(ImplicitDict): class SequenceViewConfiguration(ImplicitDict): - output_path: str - """Path of a folder into which report HTML files should be written""" + redact_access_tokens: bool = True + """When True, look for instances of "Authorization" keys in the report with values starting "Bearer " and redact the signature from those access tokens""" class ReportHTMLConfiguration(ImplicitDict): - html_path: str - """Path of HTML file to contain an HTML rendering of the raw test report object""" + redact_access_tokens: bool = True + """When True, look for instances of "Authorization" keys in the report with values starting "Bearer " and redact the signature from those access tokens""" class TemplatedReportInjectedConfiguration(ImplicitDict): @@ -69,44 +64,36 @@ class TemplatedReportConfiguration(ImplicitDict): template_url: str """Url of the template to download from""" - output_path: str - """Path of HTML file to contain the rendered templated report""" + report_name: str + """Name of HTML file (without extension) to contain the rendered templated report""" configuration: Optional[TemplatedReportInjectedConfiguration] = None """Configuration to be injected in the templated report""" -class GraphConfiguration(ImplicitDict): - gv_path: str - """Path of GraphViz (.gv) text file to contain a visualization of the test run""" - +class RawReportConfiguration(ImplicitDict): + redact_access_tokens: bool = True + """When True, look for instances of "Authorization" keys in the report with values starting "Bearer " and redact the signature from those access tokens""" -class ReportConfiguration(ImplicitDict): - report_path: str - """File name of the report to write (if test_config provided) or read (if test_config not provided)""" + indent: Optional[int] = None + """To pretty-print JSON content, specify an indent level (generally 2), or omit or set to None to write compactly.""" class ArtifactsConfiguration(ImplicitDict): - redact_access_tokens: bool = True - """When True, look for instances of "Authorization" keys in the report with values starting "Bearer " and redact the signature from those access tokens""" + output_path: str + """Path to folder where artifacts should be written.""" - report: Optional[ReportConfiguration] = None - """Configuration for report generation""" + raw_report: Optional[RawReportConfiguration] = None + """Configuration for raw report generation""" report_html: Optional[ReportHTMLConfiguration] = None - """If specified, configuration describing how an HTML version of the report should be generated""" + """If specified, configuration describing how an HTML version of the raw report should be generated""" - templated_reports: List[TemplatedReportConfiguration] = [] + templated_reports: Optional[List[TemplatedReportConfiguration]] = None """List of report templates to be rendered""" - graph: Optional[GraphConfiguration] = None - """If specified, configuration describing a desired graph visualization summarizing the test run""" - - tested_roles: Optional[TestedRolesConfiguration] = None - """If specified, configuration describing a desired report summarizing tested requirements for each specified participant and role""" - - tested_requirements: Optional[TestedRequirementsConfiguration] = None - """If specified, configuration describing a desired report summarizing all tested requirements for each participant""" + tested_requirements: Optional[List[TestedRequirementsConfiguration]] = None + """If specified, list of configurations describing desired reports summarizing tested requirements for each participant""" sequence_view: Optional[SequenceViewConfiguration] = None """If specified, configuration describing a desired report describing the sequence of events that occurred during the test""" diff --git a/monitoring/uss_qualifier/configurations/dev/dss_probing.yaml b/monitoring/uss_qualifier/configurations/dev/dss_probing.yaml index 6b458f94b0..af807b8ab7 100644 --- a/monitoring/uss_qualifier/configurations/dev/dss_probing.yaml +++ b/monitoring/uss_qualifier/configurations/dev/dss_probing.yaml @@ -26,19 +26,20 @@ v1: service_area: kentland_service_area problematically_big_area: kentland_problematically_big_area artifacts: - report: - report_path: output/report_dss_probing.json + output_path: output/dss_probing + raw_report: {} + sequence_view: {} tested_requirements: - output_path: output/tested_requirements_dss_probing - requirement_collections: - all_astm_dss_requirements: - requirement_collections: - - requirement_sets: - - astm.f3411.v22a.dss_provider - - astm.f3411.v19.dss_provider - participant_requirements: - uss1: all_astm_dss_requirements - uss2: all_astm_dss_requirements + - report_name: requirements + requirement_collections: + all_astm_dss_requirements: + requirement_collections: + - requirement_sets: + - astm.f3411.v22a.dss_provider + - astm.f3411.v19.dss_provider + participant_requirements: + uss1: all_astm_dss_requirements + uss2: all_astm_dss_requirements validation: criteria: - full_success: {} diff --git a/monitoring/uss_qualifier/configurations/dev/f3548_self_contained.yaml b/monitoring/uss_qualifier/configurations/dev/f3548_self_contained.yaml index 413fbf3cfc..cc74bc3a10 100644 --- a/monitoring/uss_qualifier/configurations/dev/f3548_self_contained.yaml +++ b/monitoring/uss_qualifier/configurations/dev/f3548_self_contained.yaml @@ -93,23 +93,23 @@ v1: # relative to where uss_qualifier is executed from, and are located inside the # Docker container executing uss_qualifier. artifacts: + # Write artifacts to this folder (relative to uss_qualifier) + output_path: output/f3548 + # Write out full report content - report: - # Path to main report output - report_path: output/report_f3548_self_contained.json + raw_report: {} - # Write out a human-readable report of the requirements tested + # Write out a human-readable report of the F3548-21 requirements tested tested_requirements: - output_path: output/tested_requirements_f3548_self_contained - requirement_collections: - scd: - requirement_collections: - - requirement_sets: - - astm.f3548.v21.scd - participant_requirements: - uss1: scd - uss2: scd + - report_name: requirements + requirement_collections: + scd: + requirement_collections: + - requirement_sets: + - astm.f3548.v21.scd + participant_requirements: + uss1: scd + uss2: scd # Write out a human-readable report showing the sequence of events of the test - sequence_view: - output_path: output/sequence_f3548_self_contained + sequence_view: {} diff --git a/monitoring/uss_qualifier/configurations/dev/general_flight_auth.yaml b/monitoring/uss_qualifier/configurations/dev/general_flight_auth.yaml index f9126e75d4..eda18dbd56 100644 --- a/monitoring/uss_qualifier/configurations/dev/general_flight_auth.yaml +++ b/monitoring/uss_qualifier/configurations/dev/general_flight_auth.yaml @@ -10,7 +10,7 @@ v1: resources: table: example_flight_check_table artifacts: - report: - report_path: output/report_general_flight_auth.json + output_path: output/general_flight_auth + raw_report: {} validation: $ref: ./library/validation.yaml#/normal_test diff --git a/monitoring/uss_qualifier/configurations/dev/geoawareness_cis.yaml b/monitoring/uss_qualifier/configurations/dev/geoawareness_cis.yaml index e44ba54145..373b631fea 100644 --- a/monitoring/uss_qualifier/configurations/dev/geoawareness_cis.yaml +++ b/monitoring/uss_qualifier/configurations/dev/geoawareness_cis.yaml @@ -13,9 +13,7 @@ v1: resources: source_document: source_document artifacts: - report: - report_path: output/report_geoawareness_cis.json - graph: - gv_path: output/report_geoawareness_cis.gv + output_path: output/geoawareness_cis + raw_report: {} validation: $ref: ./library/validation.yaml#/normal_test diff --git a/monitoring/uss_qualifier/configurations/dev/geospatial_comprehension.yaml b/monitoring/uss_qualifier/configurations/dev/geospatial_comprehension.yaml index d96d005596..d48799efb5 100644 --- a/monitoring/uss_qualifier/configurations/dev/geospatial_comprehension.yaml +++ b/monitoring/uss_qualifier/configurations/dev/geospatial_comprehension.yaml @@ -10,7 +10,7 @@ v1: resources: table: example_feature_check_table artifacts: - report: - report_path: output/report_geospatial_comprehension.json + output_path: output/geospatial_comprehension + raw_report: {} validation: $ref: ./library/validation.yaml#/normal_test diff --git a/monitoring/uss_qualifier/configurations/dev/library/resources.yaml b/monitoring/uss_qualifier/configurations/dev/library/resources.yaml index 5837c2da8f..bdc07830bd 100644 --- a/monitoring/uss_qualifier/configurations/dev/library/resources.yaml +++ b/monitoring/uss_qualifier/configurations/dev/library/resources.yaml @@ -101,13 +101,13 @@ adjacent_circular_storage_config: $content_schema: monitoring/uss_qualifier/resources/definitions/ResourceDeclaration.json resource_type: resources.netrid.FlightDataStorageResource specification: - flight_record_collection_path: "./output/test_data.che.netrid.circular_flights.json" + flight_record_collection_path: "./output/generate_rid_test_data/flight_data/test_data.che.netrid.circular_flights.json" kml_storage_config: $content_schema: monitoring/uss_qualifier/resources/definitions/ResourceDeclaration.json resource_type: resources.netrid.FlightDataStorageResource specification: - flight_record_collection_path: "./output/test_data.usa.netrid.dcdemo_flights.json" + flight_record_collection_path: "./output/generate_rid_test_data/flight_data/test_data.usa.netrid.dcdemo_flights.json" # ===== Flight planning intents ===== diff --git a/monitoring/uss_qualifier/configurations/dev/message_signing.yaml b/monitoring/uss_qualifier/configurations/dev/message_signing.yaml index ea33cc7af7..dfb2ec8491 100644 --- a/monitoring/uss_qualifier/configurations/dev/message_signing.yaml +++ b/monitoring/uss_qualifier/configurations/dev/message_signing.yaml @@ -50,5 +50,5 @@ v1: dss: scd_dss artifacts: - report: - report_path: output/report_message_signing.json + output_path: output/message_signing + raw_report: {} diff --git a/monitoring/uss_qualifier/configurations/dev/netrid_v19.yaml b/monitoring/uss_qualifier/configurations/dev/netrid_v19.yaml index 65e3b32062..3a63bd3cb3 100644 --- a/monitoring/uss_qualifier/configurations/dev/netrid_v19.yaml +++ b/monitoring/uss_qualifier/configurations/dev/netrid_v19.yaml @@ -31,26 +31,23 @@ v1: service_area: kentland_service_area problematically_big_area: au_problematically_big_area artifacts: - report: - report_path: output/report_netrid_v19.json - tested_roles: - report_path: output/tested_roles_netrid_v19 + output_path: output/netrid_v19 + raw_report: {} tested_requirements: - output_path: output/tested_requirements_f3411v19 - requirement_collections: - sp_dp_dss: - requirement_sets: - - astm.f3411.v19.service_provider#Tested by automated tests - - astm.f3411.v19.display_provider#Automated verification - - astm.f3411.v19.dss_provider - sp_dss: - requirement_sets: - - astm.f3411.v19.service_provider#Tested by automated tests - - astm.f3411.v19.dss_provider - participant_requirements: - uss1: sp_dp_dss - uss2: sp_dss - sequence_view: - output_path: output/sequence_netrid_v19 + - report_name: requirements + requirement_collections: + sp_dp_dss: + requirement_sets: + - astm.f3411.v19.service_provider#Tested by automated tests + - astm.f3411.v19.display_provider#Automated verification + - astm.f3411.v19.dss_provider + sp_dss: + requirement_sets: + - astm.f3411.v19.service_provider#Tested by automated tests + - astm.f3411.v19.dss_provider + participant_requirements: + uss1: sp_dp_dss + uss2: sp_dss + sequence_view: {} validation: $ref: ./library/validation.yaml#/normal_test diff --git a/monitoring/uss_qualifier/configurations/dev/netrid_v22a.yaml b/monitoring/uss_qualifier/configurations/dev/netrid_v22a.yaml index 7d919e2ebd..5441555c5a 100644 --- a/monitoring/uss_qualifier/configurations/dev/netrid_v22a.yaml +++ b/monitoring/uss_qualifier/configurations/dev/netrid_v22a.yaml @@ -31,26 +31,23 @@ v1: service_area: kentland_service_area problematically_big_area: au_problematically_big_area artifacts: - report: - report_path: output/report_netrid_v22a.json - tested_roles: - report_path: output/tested_roles_netrid_v22a + output_path: output/netrid_v22a + raw_report: {} tested_requirements: - output_path: output/tested_requirements_f3411v22a - requirement_collections: - sp_dp_dss: - requirement_sets: - - astm.f3411.v22a.service_provider#Mandatory requirements - - astm.f3411.v22a.display_provider#Mandatory requirements - - astm.f3411.v22a.dss_provider - sp_dss: - requirement_sets: - - astm.f3411.v22a.service_provider#Mandatory requirements - - astm.f3411.v22a.dss_provider - participant_requirements: - uss1: sp_dp_dss - uss2: sp_dss - sequence_view: - output_path: output/sequence_netrid_v22a + - report_name: requirements + requirement_collections: + sp_dp_dss: + requirement_sets: + - astm.f3411.v22a.service_provider#Mandatory requirements + - astm.f3411.v22a.display_provider#Mandatory requirements + - astm.f3411.v22a.dss_provider + sp_dss: + requirement_sets: + - astm.f3411.v22a.service_provider#Mandatory requirements + - astm.f3411.v22a.dss_provider + participant_requirements: + uss1: sp_dp_dss + uss2: sp_dss + sequence_view: {} validation: $ref: ./library/validation.yaml#/normal_test diff --git a/monitoring/uss_qualifier/configurations/dev/noop.yaml b/monitoring/uss_qualifier/configurations/dev/noop.yaml index 95af477c6d..be427560a8 100644 --- a/monitoring/uss_qualifier/configurations/dev/noop.yaml +++ b/monitoring/uss_qualifier/configurations/dev/noop.yaml @@ -24,8 +24,10 @@ v1: noop_config: noop_config artifacts: - report: - # Path to main report output - report_path: output/report_noop.json + output_path: output/noop + raw_report: + indent: 2 + sequence_view: {} + report_html: {} validation: $ref: ./library/validation.yaml#/normal_test diff --git a/monitoring/uss_qualifier/configurations/dev/uspace.yaml b/monitoring/uss_qualifier/configurations/dev/uspace.yaml index aa8c389757..8aa10486c9 100644 --- a/monitoring/uss_qualifier/configurations/dev/uspace.yaml +++ b/monitoring/uss_qualifier/configurations/dev/uspace.yaml @@ -83,41 +83,38 @@ v1: service_area: service_area problematically_big_area: problematically_big_area artifacts: - tested_roles: - report_path: output/tested_roles_uspace - report: - report_path: output/report_uspace.json + output_path: output/uspace + raw_report: {} templated_reports: - template_url: https://github.com/Orbitalize/reports/releases/download/v0.0.18/app-v0.0.18.zip - output_path: output/capabilities_uspace.html + report_name: capabilities tested_requirements: - output_path: output/tested_requirements_uspace - requirement_collections: - uspace: - requirement_collections: - - requirement_sets: - - astm.f3411.v22a.service_provider#Mandatory requirements - - astm.f3411.v22a.service_provider#Operator ID provider - - astm.f3411.v22a.service_provider#UAS ID Serial Number provider - - astm.f3411.v22a.service_provider#Height provider - - astm.f3411.v22a.service_provider#Operator Position provider - - astm.f3411.v22a.service_provider#Operational Status provider - - astm.f3411.v22a.display_provider#Mandatory requirements - - astm.f3411.v22a.display_provider#UAS ID Serial Number transmitter - - astm.f3411.v22a.display_provider#Timestamp transmitter - - astm.f3411.v22a.display_provider#Operational Status transmitter - - astm.f3411.v22a.display_provider#Operator ID transmitter - - astm.f3411.v22a.display_provider#Current Position transmitter - - astm.f3411.v22a.display_provider#Height transmitter - - astm.f3411.v22a.display_provider#Track Direction transmitter - - astm.f3411.v22a.display_provider#Speed transmitter - - astm.f3411.v22a.display_provider#Operator Position transmitter - - astm.f3411.v22a.dss_provider - - astm.f3548.v21.scd#Automated verification - participant_requirements: - uss1: uspace - uss2: uspace - sequence_view: - output_path: output/sequence_uspace + - report_name: requirements + requirement_collections: + uspace: + requirement_collections: + - requirement_sets: + - astm.f3411.v22a.service_provider#Mandatory requirements + - astm.f3411.v22a.service_provider#Operator ID provider + - astm.f3411.v22a.service_provider#UAS ID Serial Number provider + - astm.f3411.v22a.service_provider#Height provider + - astm.f3411.v22a.service_provider#Operator Position provider + - astm.f3411.v22a.service_provider#Operational Status provider + - astm.f3411.v22a.display_provider#Mandatory requirements + - astm.f3411.v22a.display_provider#UAS ID Serial Number transmitter + - astm.f3411.v22a.display_provider#Timestamp transmitter + - astm.f3411.v22a.display_provider#Operational Status transmitter + - astm.f3411.v22a.display_provider#Operator ID transmitter + - astm.f3411.v22a.display_provider#Current Position transmitter + - astm.f3411.v22a.display_provider#Height transmitter + - astm.f3411.v22a.display_provider#Track Direction transmitter + - astm.f3411.v22a.display_provider#Speed transmitter + - astm.f3411.v22a.display_provider#Operator Position transmitter + - astm.f3411.v22a.dss_provider + - astm.f3548.v21.scd#Automated verification + participant_requirements: + uss1: uspace + uss2: uspace + sequence_view: {} validation: $ref: ./library/validation.yaml#/normal_test diff --git a/monitoring/uss_qualifier/main.py b/monitoring/uss_qualifier/main.py index 900ca905f6..8351de8b2c 100644 --- a/monitoring/uss_qualifier/main.py +++ b/monitoring/uss_qualifier/main.py @@ -14,7 +14,7 @@ from monitoring.uss_qualifier.configurations.configuration import ( USSQualifierConfiguration, ArtifactsConfiguration, - ReportConfiguration, + RawReportConfiguration, USSQualifierConfigurationV1, ) from monitoring.uss_qualifier.fileio import load_dict_with_references @@ -23,8 +23,6 @@ from monitoring.uss_qualifier.reports.tested_requirements import ( generate_tested_requirements, ) -from monitoring.uss_qualifier.reports.tested_roles import generate_tested_roles -from monitoring.uss_qualifier.reports.graphs import make_graph from monitoring.uss_qualifier.reports.report import TestRunReport, redact_access_tokens from monitoring.uss_qualifier.reports.templates import render_templates from monitoring.uss_qualifier.reports.validation.report_validation import ( @@ -159,10 +157,10 @@ def run_config( if report_path: if not config.artifacts: config.artifacts = ArtifactsConfiguration( - ReportConfiguration(report_path=report_path) + RawReportConfiguration(report_path=report_path) ) elif not config.artifacts.report: - config.artifacts.report = ReportConfiguration(report_path=report_path) + config.artifacts.report = RawReportConfiguration(report_path=report_path) else: config.artifacts.report.report_path = report_path @@ -180,46 +178,70 @@ def run_config( ) if config.artifacts: - if config.artifacts.report and not do_not_save_report: - if config.artifacts.redact_access_tokens: - logger.info("Redacting access tokens in report") - redact_access_tokens(report) - logger.info("Writing report to {}", config.artifacts.report.report_path) - with open(config.artifacts.report.report_path, "w") as f: - json.dump(report, f, indent=2) + os.makedirs(config.artifacts.output_path, exist_ok=True) - if config.artifacts.report_html: - logger.info( - "Writing HTML report to {}", config.artifacts.report_html.html_path - ) - with open(config.artifacts.report_html.html_path, "w") as f: - f.write(make_report_html(report)) - - if len(config.artifacts.templated_reports) > 0: + def _should_redact(cfg) -> bool: + return "redact_access_tokens" in cfg and cfg.redact_access_tokens - render_templates(config.artifacts, report) + logger.info(f"Redacting access tokens from report") + redacted_report = ImplicitDict.parse( + json.loads(json.dumps(report)), TestRunReport + ) + redact_access_tokens(redacted_report) + + if config.artifacts.raw_report and not do_not_save_report: + # Raw report + path = os.path.join(config.artifacts.output_path, "report.json") + logger.info(f"Writing raw report to {path}") + raw_report = config.artifacts.raw_report + report_to_write = redacted_report if _should_redact(raw_report) else report + with open(path, "w") as f: + if "indent" in raw_report and raw_report.indent is not None: + json.dump(report_to_write, f, indent=raw_report.indent) + else: + json.dump(report_to_write, f) - if config.artifacts.graph: - logger.info( - "Writing GraphViz dot source to {}", config.artifacts.graph.gv_path + if config.artifacts.report_html: + # HTML rendering of raw report + path = os.path.join(config.artifacts.output_path, "report.html") + logger.info(f"Writing HTML report to {path}") + report_to_write = ( + redacted_report + if _should_redact(config.artifacts.report_html) + else report + ) + with open(path, "w") as f: + f.write(make_report_html(report_to_write)) + + if config.artifacts.templated_reports: + # Templated reports + render_templates( + config.artifacts.output_path, + config.artifacts.templated_reports, + redacted_report, ) - with open(config.artifacts.graph.gv_path, "w") as f: - f.write(make_graph(report).source) - - if config.artifacts.tested_roles: - path = config.artifacts.tested_roles.report_path - logger.info("Writing tested roles view to {}", path) - generate_tested_roles(report, path) if config.artifacts.tested_requirements: - path = config.artifacts.tested_requirements.output_path - logger.info(f"Writing tested requirements view to {path}") - generate_tested_requirements(report, config.artifacts.tested_requirements) + # Tested requirements view + for tested_reqs_config in config.artifacts.tested_requirements: + path = os.path.join( + config.artifacts.output_path, tested_reqs_config.report_name + ) + logger.info(f"Writing tested requirements view to {path}") + generate_tested_requirements(redacted_report, tested_reqs_config, path) if config.artifacts.sequence_view: - path = config.artifacts.sequence_view.output_path + # Sequence view + path = os.path.join(config.artifacts.output_path, "sequence") logger.info(f"Writing sequence view to {path}") - generate_sequence_view(report, config.artifacts.sequence_view) + report_to_write = ( + redacted_report + if _should_redact(config.artifacts.sequence_view) + else report + ) + generate_sequence_view( + report_to_write, config.artifacts.sequence_view, path + ) if "validation" in config and config.validation: logger.info(f"Validating test run report for configuration '{config_name}'") diff --git a/monitoring/uss_qualifier/reports/graphs.py b/monitoring/uss_qualifier/reports/graphs.py deleted file mode 100644 index a9264faee8..0000000000 --- a/monitoring/uss_qualifier/reports/graphs.py +++ /dev/null @@ -1,317 +0,0 @@ -from typing import Optional, List, Set, Tuple, Dict - -import graphviz -from loguru import logger - -from implicitdict import ImplicitDict - -from monitoring.uss_qualifier.reports.report import ( - ActionGeneratorReport, - TestRunReport, - TestSuiteReport, - TestScenarioReport, -) -from monitoring.uss_qualifier.resources.definitions import ( - ResourceID, - ResourceCollection, -) -from monitoring.uss_qualifier.scenarios.definitions import TestScenarioDeclaration -from monitoring.uss_qualifier.suites.definitions import ( - ActionType, - TestSuiteDeclaration, - TestSuiteDefinition, - ActionGeneratorDefinition, -) - -NodeName = str - - -class Node(ImplicitDict): - """Represents a node to be used in a GraphViz graph.""" - - name: NodeName - label: Optional[str] = None - children: List[NodeName] - attributes: Dict[str, str] - - -class NodeNamer(object): - """Creates and tracks unique, valid names within a GraphViz graph.""" - - names: Set[NodeName] = set() - - def make_name(self, desired_name: NodeName) -> NodeName: - acceptable_name = desired_name.replace(" ", "").replace(".", "_") - actual_name = acceptable_name - i = 2 - while actual_name in self.names: - actual_name = f"{acceptable_name}_{i}" - i += 1 - self.names.add(actual_name) - return actual_name - - def use_name(self, name: NodeName) -> None: - self.names.add(name) - - -def _make_test_scenario_nodes( - declaration: Optional[TestScenarioDeclaration], - report: TestScenarioReport, - nodes_by_id: Dict[ResourceID, Node], - namer: NodeNamer, - include_notes: bool = False, -) -> List[Node]: - nodes: List[Node] = [] - - # Make the scenario node - scenario_type = report.scenario_type - if scenario_type.startswith("scenarios."): - scenario_type = scenario_type[len("scenarios.") :] - label_elements = [report.name, scenario_type] - if include_notes and "notes" in report: - label_elements += [f"{k}={v.message}" for k, v in report.notes.items()] - scenario_node = Node( - name=namer.make_name(report.scenario_type), - label="\n".join(label_elements), - children=[], - attributes={ - "shape": "component", - "fillcolor": "lightgreen" if report.successful else "lightpink", - "style": "filled", - }, - ) - - # Mark the scenario node a child of the appropriate resources - if declaration is not None: - for local_id, node in nodes_by_id.items(): - node.children.append(scenario_node.name) - - # Add failed checks and error below - parent_node = scenario_node - for _, failed_check in report.query_failed_checks(): - check_node = Node( - name=namer.make_name(report.scenario_type + "FailedCheck"), - label=failed_check.summary, - children=[], - attributes={ - "shape": "octagon", - "fillcolor": "lightpink", - "style": "filled", - }, - ) - nodes.append(check_node) - parent_node.children.append(check_node.name) - parent_node = check_node - if "execution_error" in report: - error_node = Node( - name=namer.make_name(report.scenario_type + "ExecutionError"), - label=report.execution_error.message, - children=[], - attributes={ - "shape": "octagon", - "fillcolor": "lightpink", - "style": "filled", - "color": "red", - }, - ) - nodes.append(error_node) - parent_node.children.append(error_node.name) - - # Add the scenario node last - nodes.append(scenario_node) - - return nodes - - -def _make_test_suite_nodes( - declaration: TestSuiteDeclaration, - report: TestSuiteReport, - nodes_by_id: Dict[ResourceID, Node], - namer: NodeNamer, -) -> List[Node]: - # Make child nodes for each action in the suite - new_nodes = None - nodes: List[Node] = [] - children: List[NodeName] = [] - definition = TestSuiteDefinition.load_from_declaration(declaration) - for action_report, action in zip(report.actions, definition.actions): - action_nodes = _translate_ids(nodes_by_id, action.get_resource_links()) - if "test_suite" in action_report: - new_nodes = _make_test_suite_nodes( - action.test_suite, - action_report.test_suite, - action_nodes, - namer, - ) - children.append(new_nodes[-1].name) - elif "test_scenario" in action_report: - new_nodes = _make_test_scenario_nodes( - action.test_scenario, action_report.test_scenario, action_nodes, namer - ) - nodes.extend(new_nodes) - children.append(new_nodes[-1].name) - elif "action_generator" in action_report: - new_nodes = _make_action_generator_nodes( - action.action_generator, - action_report.action_generator, - action_nodes, - namer, - ) - children.append(new_nodes[-1].name) - else: - ActionType.raise_invalid_action_declaration() - nodes.extend(new_nodes) - - # Make the suite node itself - suite_type = report.suite_type - if suite_type.startswith("suites."): - suite_type = suite_type[len("suites.") :] - suite_node = Node( - name=namer.make_name(report.suite_type), - label=report.name + "\n" + suite_type, - children=children, - attributes={"shape": "folder"}, - ) - nodes.append(suite_node) - return nodes - - -def _make_action_generator_nodes( - definition: ActionGeneratorDefinition, - report: ActionGeneratorReport, - nodes_by_id: Dict[ResourceID, Node], - namer: NodeNamer, -) -> List[Node]: - # Make the action generator node - nodes: List[Node] = [] - children: List[NodeName] = [] - for action in report.actions: - if "test_suite" in action: - # TODO: Support visualization of test suite actions with an action generator - logger.warning( - "test_suite action is being omitted from the action generator because graph representation of a test suite from an action generator is not yet supported" - ) - new_nodes = [] - elif "test_scenario" in action: - new_nodes = _make_test_scenario_nodes( - None, action.test_scenario, nodes_by_id, namer, True - ) - nodes.extend(new_nodes) - children.append(new_nodes[-1].name) - elif "action_generator" in action: - raise NotImplementedError() - else: - raise NotImplementedError() - nodes.extend(new_nodes) - generator_type = report.generator_type - if generator_type.startswith("action_generators."): - generator_type = generator_type[len("action_generators.") :] - generator_node = Node( - name=namer.make_name(report.generator_type), - label="Action generator\n" + generator_type, - children=children, - attributes={"shape": "box3d"}, - ) - nodes.append(generator_node) - - # Point all resources used by children of the action generator to the action generator - for local_id, node in _translate_ids(nodes_by_id, definition.resources).items(): - node.children.append(generator_node.name) - - return nodes - - -def _make_resource_nodes( - resources: ResourceCollection, namer: NodeNamer -) -> Tuple[List[Node], Dict[ResourceID, Node]]: - nodes: List[Node] = [] - nodes_by_id: Dict[ResourceID, Node] = {} - added = 1 - while added > 0: - added = 0 - for resource_id, declaration in resources.resource_declarations.items(): - # Skip resources that already have nodes - if resource_id in namer.names: - continue - - # Identify prerequisite resources not yet created - prereqs = [] - for child_param, parent_id in declaration.dependencies.items(): - if parent_id not in nodes_by_id: - prereqs.append(parent_id) - - # Create this resource node if there are no uncreated prerequisites - if not prereqs: - resource_type = declaration.resource_type - if resource_type.startswith("resources."): - resource_type = resource_type[len("resources.") :] - resource_node = Node( - name=namer.make_name(resource_id), - label=f"{resource_id}\n{resource_type}", - children=[], - attributes={ - "shape": "note", - "fillcolor": "lightskyblue1", - "style": "filled", - }, - ) - namer.use_name(resource_id) - nodes.append(resource_node) - nodes_by_id[resource_id] = resource_node - for child_param, parent_id in declaration.dependencies.items(): - nodes_by_id[parent_id].children.append(resource_node.name) - added += 1 - return nodes, nodes_by_id - - -def _translate_ids( - nodes_by_id: Dict[ResourceID, Node], local_resources: Dict[ResourceID, ResourceID] -) -> Dict[ResourceID, Node]: - local_id_by_parent_id = {v: k for k, v in local_resources.items()} - return { - local_id_by_parent_id[parent_id]: node - for parent_id, node in nodes_by_id.items() - if parent_id in local_id_by_parent_id - } - - -def make_graph(report: TestRunReport) -> graphviz.Digraph: - namer = NodeNamer() - - # Make nodes for resources - nodes, nodes_by_id = _make_resource_nodes(report.configuration.resources, namer) - - action_type = report.configuration.action.get_action_type() - if action_type == ActionType.TestSuite: - test_suite = report.configuration.action.test_suite - test_suite_report = report.report.test_suite - - # Translate resource names into the action frame - suite_nodes_by_id = _translate_ids(nodes_by_id, test_suite.resources) - - # Make nodes for the suite - nodes.extend( - _make_test_suite_nodes( - test_suite, test_suite_report, suite_nodes_by_id, namer - ) - ) - elif action_type == ActionType.TestScenario: - test_scenario = report.configuration.action.test_scenario - test_scenario_report = report.report.test_scenario - scenario_nodes_by_id = _translate_ids(nodes_by_id, test_scenario.resources) - nodes.extend( - _make_test_scenario_nodes( - test_scenario, test_scenario_report, scenario_nodes_by_id, namer - ) - ) - else: - raise NotImplementedError() - - # Translate nodes into GraphViz - dot = graphviz.Digraph(node_attr={"shape": "box"}) - for node in nodes: - dot.node(name=node.name, label=node.label, **node.attributes) - for child in node.children: - dot.edge(node.name, child) - - return dot diff --git a/monitoring/uss_qualifier/reports/sequence_view.py b/monitoring/uss_qualifier/reports/sequence_view.py index cba2d75efa..c87b3f52a4 100644 --- a/monitoring/uss_qualifier/reports/sequence_view.py +++ b/monitoring/uss_qualifier/reports/sequence_view.py @@ -597,7 +597,7 @@ def _enumerate_all_participants(node: ActionNode) -> List[ParticipantID]: def _generate_scenario_pages( - node: ActionNode, config: SequenceViewConfiguration + node: ActionNode, config: SequenceViewConfiguration, output_path: str ) -> None: if node.node_type == ActionNodeType.Scenario: all_participants = list(node.scenario.participants) @@ -606,7 +606,7 @@ def _generate_scenario_pages( all_participants.remove(UNATTRIBUTED_PARTICIPANT) all_participants.append(UNATTRIBUTED_PARTICIPANT) scenario_file = os.path.join( - config.output_path, f"s{node.scenario.scenario_index}.html" + output_path, f"s{node.scenario.scenario_index}.html" ) template = jinja_env.get_template("sequence_view/scenario.html") with open(scenario_file, "w") as f: @@ -623,16 +623,16 @@ def _generate_scenario_pages( ) else: for child in node.children: - _generate_scenario_pages(child, config) + _generate_scenario_pages(child, config, output_path) def generate_sequence_view( - report: TestRunReport, config: SequenceViewConfiguration + report: TestRunReport, config: SequenceViewConfiguration, output_path: str ) -> None: node = _compute_action_node(report.report, Indexer()) - os.makedirs(config.output_path, exist_ok=True) - _generate_scenario_pages(node, config) + os.makedirs(output_path, exist_ok=True) + _generate_scenario_pages(node, config, output_path) overview_rows = list(_compute_overview_rows(node)) _align_overview_rows(overview_rows) @@ -642,7 +642,7 @@ def generate_sequence_view( if UNATTRIBUTED_PARTICIPANT in all_participants: all_participants.remove(UNATTRIBUTED_PARTICIPANT) all_participants.append(UNATTRIBUTED_PARTICIPANT) - overview_file = os.path.join(config.output_path, "index.html") + overview_file = os.path.join(output_path, "index.html") template = jinja_env.get_template("sequence_view/overview.html") with open(overview_file, "w") as f: f.write( diff --git a/monitoring/uss_qualifier/reports/templates.py b/monitoring/uss_qualifier/reports/templates.py index 5a1b36ae66..5ac2a05472 100644 --- a/monitoring/uss_qualifier/reports/templates.py +++ b/monitoring/uss_qualifier/reports/templates.py @@ -1,6 +1,7 @@ import json import shutil import os +from typing import List from loguru import logger @@ -57,10 +58,12 @@ def _download_template(self) -> pathlib.Path: logger.debug(f"{url} extracted to {path}") return path - def render(self): + def render(self, base_path: str): # Copy template src = pathlib.Path(self._download_template(), "index.html") - dst = pathlib.Path(self._template.output_path) + dst = pathlib.Path( + os.path.join(base_path, self._template.report_name + ".html") + ) # Configure application rendered_configuration = json.dumps( @@ -82,7 +85,11 @@ def render(self): logger.info(f"Templated report rendered to {dst}") -def render_templates(config: ArtifactsConfiguration, report: TestRunReport): +def render_templates( + base_path: str, + templated_reports: List[TemplatedReportConfiguration], + report: TestRunReport, +): pathlib.Path(CACHE_TEMPLATE_PATH).mkdir(parents=True, exist_ok=True) - for template in config.templated_reports: - TemplateRenderer(template, report).render() + for template in templated_reports: + TemplateRenderer(template, report).render(base_path) diff --git a/monitoring/uss_qualifier/reports/templates/tested_roles/capability_evaluation_report.html b/monitoring/uss_qualifier/reports/templates/tested_roles/capability_evaluation_report.html deleted file mode 100644 index b057e70fa8..0000000000 --- a/monitoring/uss_qualifier/reports/templates/tested_roles/capability_evaluation_report.html +++ /dev/null @@ -1,101 +0,0 @@ - - - - - -
-

{{ test_suite_name }} test suite

- - - - - - {% for capability in capabilities %} - - {% if capability.requirements %} - - - - - - - - {% for req in capability.requirements %} - {% for pc in req.passed_checks %} - - {% if loop.index == 1 %} - - {% endif %} - - - - - - {% endfor %} - {% for fc in req.failed_checks %} - - {% if loop.index == 1 %} - - {% endif %} - - - - - - {% endfor %} - {% if (len(req.passed_checks) == 0) and (len(req.failed_checks) == 0) %} - - - - - {% endif %} - {% endfor %} - {% endif %} - - {% if capability.capabilities %} - - {% if not capability.requirements %} - - {% endif %} - - - - - {% for child_cap in capability.capabilities %} - - - - - {% endfor %} - {% endif %} - - {% endfor %} -
CapabilityEvaluation
{{ capability.name }}RequirementTest checkResult
{{ req.requirement_id }}{{ pc }} ({{ req.passed_checks[pc] }}x)PASS
{{ req.requirement_id }}{{ fc.name }}FAIL
{{ req.requirement_id }}Not tested
{{ capability.name }}CapabilityResult
{{ child_cap.capability_id }}{{ 'VERIFIED' if child_cap.verified else 'MISSING' if child_cap.missing else 'UNVERIFIED' }}
-
- - diff --git a/monitoring/uss_qualifier/reports/templates/tested_roles/test_run_report.html b/monitoring/uss_qualifier/reports/templates/tested_roles/test_run_report.html deleted file mode 100644 index 3698a92c4e..0000000000 --- a/monitoring/uss_qualifier/reports/templates/tested_roles/test_run_report.html +++ /dev/null @@ -1,12 +0,0 @@ - - -
-

Participants

- -
- - diff --git a/monitoring/uss_qualifier/reports/tested_requirements.py b/monitoring/uss_qualifier/reports/tested_requirements.py index 2ec61863df..4bc4cfb2df 100644 --- a/monitoring/uss_qualifier/reports/tested_requirements.py +++ b/monitoring/uss_qualifier/reports/tested_requirements.py @@ -1,6 +1,5 @@ import os from dataclasses import dataclass -import re from functools import cmp_to_key from typing import List, Union, Dict, Set, Optional @@ -187,7 +186,7 @@ class TestRunInformation(object): def generate_tested_requirements( - report: TestRunReport, config: TestedRequirementsConfiguration + report: TestRunReport, config: TestedRequirementsConfiguration, output_path: str ) -> None: req_collections: Dict[ TestedRequirementsCollectionIdentifier, Set[RequirementID] @@ -211,8 +210,8 @@ def generate_tested_requirements( import_submodules(suites) import_submodules(action_generators) - os.makedirs(config.output_path, exist_ok=True) - index_file = os.path.join(config.output_path, "index.html") + os.makedirs(output_path, exist_ok=True) + index_file = os.path.join(output_path, "index.html") participant_ids = list(report.report.participant_ids()) participant_ids.sort() @@ -237,7 +236,7 @@ def generate_tested_requirements( participant_breakdown, participant_req_collections[participant_id] ) _sort_breakdown(participant_breakdown) - participant_file = os.path.join(config.output_path, f"{participant_id}.html") + participant_file = os.path.join(output_path, f"{participant_id}.html") other_participants = ", ".join( p for p in participant_ids if p != participant_id ) diff --git a/monitoring/uss_qualifier/reports/tested_roles.py b/monitoring/uss_qualifier/reports/tested_roles.py deleted file mode 100644 index 99192ce8ad..0000000000 --- a/monitoring/uss_qualifier/reports/tested_roles.py +++ /dev/null @@ -1,217 +0,0 @@ -import os -import shutil -from dataclasses import dataclass -from typing import List, Any, Union, Dict - -from monitoring.uss_qualifier.configurations.configuration import ParticipantID -from monitoring.uss_qualifier.reports import jinja_env -from monitoring.uss_qualifier.reports.report import ( - TestRunReport, - TestSuiteActionReport, - TestScenarioReport, - ActionGeneratorReport, - TestSuiteReport, - PassedCheck, - FailedCheck, - ParticipantCapabilityConditionEvaluationReport, -) -from monitoring.uss_qualifier.requirements.definitions import RequirementID - - -def generate_tested_roles(report: TestRunReport, output_path: str) -> None: - if os.path.exists(output_path): - shutil.rmtree(output_path) - participant_ids = report.report.participant_ids() - for participant_id in participant_ids: - _generate_suite_action_output( - report.report, participant_id, os.path.join(output_path, participant_id) - ) - template = jinja_env.get_template("tested_roles/test_run_report.html") - index = os.path.join(output_path, "index.html") - os.makedirs(os.path.dirname(index), exist_ok=True) - with open(index, "w") as f: - f.write(template.render(participant_ids=participant_ids)) - - -def _generate_suite_action_output( - report: TestSuiteActionReport, participant_id: ParticipantID, output_path: str -) -> None: - test_suite, test_scenario, action_generator = report.get_applicable_report() - if test_suite: - _generate_suite_output(report.test_suite, participant_id, output_path) - elif test_scenario: - _generate_scenario_output(report.test_scenario, participant_id, output_path) - elif action_generator: - _generate_action_generator_output( - report.action_generator, participant_id, output_path - ) - else: - raise ValueError("TestSuiteActionReport did not specify any report") - - -def _generate_scenario_output( - report: TestScenarioReport, participant_id: ParticipantID, output_path: str -) -> None: - pass - - -def _generate_action_generator_output( - report: ActionGeneratorReport, participant_id: ParticipantID, output_path: str -) -> None: - for i, action in enumerate(report.actions): - _generate_suite_action_output( - action, participant_id, os.path.join(output_path, f"action{i}") - ) - - -@dataclass -class PotentiallyCheckedRequirement(object): - requirement_id: RequirementID - passed_checks: Dict[str, int] - failed_checks: List[FailedCheck] - - @property - def rows(self) -> int: - if not self.passed_checks and not self.failed_checks: - return 1 - return len(self.passed_checks) + len(self.failed_checks) - - -@dataclass -class ChildCapability(object): - capability_id: str - verified: bool = False - missing: bool = False - - @property - def rows(self) -> int: - return 1 - - -@dataclass -class CapabilityEvalInfo(object): - requirements: List[PotentiallyCheckedRequirement] - capabilities: List[ChildCapability] - - -@dataclass -class CapabilityEvalReport(object): - name: str - verified: bool - requirements: List[PotentiallyCheckedRequirement] - capabilities: List[ChildCapability] - - @property - def rows(self) -> int: - n = 0 - if self.requirements: - n += 1 - n += sum(r.rows for r in self.requirements) - if self.capabilities: - n += 1 - n += sum(c.rows for c in self.capabilities) - return n - - -def _follow_jsonaddress(obj: dict, fields: Union[str, List[str]]) -> Any: - """Follows an explicit JSONAddress for an obj (much, much faster than treating path as a search).""" - # TODO: Change fields to be Union[JSONAddress, List[JSONAddress]] upon merging of #171 - - if isinstance(fields, str): - return _follow_jsonaddress(obj, fields.split(".")) - if not fields: - return obj - if fields[0] == "$" or fields[0] == "": - return _follow_jsonaddress(obj, fields[1:]) - - if len(fields) == 1: - field = fields[0] - if field[-1] == "]" and "[" in field: - field, index = field[0:-1].split("[") - items = _follow_jsonaddress(obj, field) - return items[int(index)] - else: - return obj[field] - else: - child = _follow_jsonaddress(obj, [fields[0]]) - return _follow_jsonaddress(child, fields[1:]) - - -def _collect_info_from_conditions( - report: TestSuiteReport, condition: ParticipantCapabilityConditionEvaluationReport -) -> CapabilityEvalInfo: - result = CapabilityEvalInfo(requirements=[], capabilities=[]) - if "all_conditions" in condition and condition.all_conditions: - for subcondition in ( - condition.all_conditions.satisfied_conditions - + condition.all_conditions.unsatisfied_conditions - ): - subresult = _collect_info_from_conditions(report, subcondition) - result.requirements.extend(subresult.requirements) - result.capabilities.extend(subresult.capabilities) - elif "requirements_checked" in condition and condition.requirements_checked: - for req_ref in ( - condition.requirements_checked.passed_requirements - + condition.requirements_checked.failed_requirements - ): - passed_checks = {} - failed_checks = [] - for req_path in req_ref.passed_checks: - pc: PassedCheck = _follow_jsonaddress(report, req_path) - passed_checks[pc.name] = passed_checks.get(pc.name, 0) + 1 - for req_path in req_ref.failed_checks: - failed_checks.append(_follow_jsonaddress(report, req_path)) - req = PotentiallyCheckedRequirement( - requirement_id=req_ref.requirement_id, - passed_checks=passed_checks, - failed_checks=failed_checks, - ) - result.requirements.append(req) - for req_id in condition.requirements_checked.untested_requirements: - result.requirements.append( - PotentiallyCheckedRequirement( - requirement_id=req_id, passed_checks={}, failed_checks=[] - ) - ) - elif "capability_verified" in condition and condition.capability_verified: - for c in condition.capability_verified.checked_capabilities: - result.capabilities.append( - ChildCapability( - capability_id=c.capability_id, verified=c.capability_verified - ) - ) - for c in condition.capability_verified.missing_capabilities: - result.capabilities.append(ChildCapability(capability_id=c, missing=True)) - return result - - -def _generate_suite_output( - report: TestSuiteReport, participant_id: ParticipantID, output_path: str -) -> None: - capabilities: List[CapabilityEvalReport] = [] - for capability in report.capability_evaluations: - if capability.participant_id != participant_id: - continue - info = _collect_info_from_conditions(report, capability.condition_evaluation) - capabilities.append( - CapabilityEvalReport( - name=capability.capability_id, - verified=capability.verified, - requirements=info.requirements, - capabilities=info.capabilities, - ) - ) - template = jinja_env.get_template("tested_roles/capability_evaluation_report.html") - index = os.path.join(output_path, "index.html") - os.makedirs(os.path.dirname(index)) - with open(index, "w") as f: - f.write( - template.render( - len=len, test_suite_name=report.name, capabilities=capabilities - ) - ) - - for i, action in enumerate(report.actions): - _generate_suite_action_output( - action, participant_id, os.path.join(output_path, f"action{i}") - ) diff --git a/monitoring/uss_qualifier/run_locally.sh b/monitoring/uss_qualifier/run_locally.sh index 4ac41fb7a5..da4423dc2a 100755 --- a/monitoring/uss_qualifier/run_locally.sh +++ b/monitoring/uss_qualifier/run_locally.sh @@ -58,7 +58,6 @@ else docker_args="-it" fi -start_time=$(date +%Y-%m-%dT%H:%M:%S) # shellcheck disable=SC2086 docker run ${docker_args} --name uss_qualifier \ --rm \ @@ -74,16 +73,3 @@ docker run ${docker_args} --name uss_qualifier \ -w /app/monitoring/uss_qualifier \ interuss/monitoring \ python main.py $QUALIFIER_OPTIONS - -# Set return code according to whether the test run was fully successful -reports_generated=$(find ./monitoring/uss_qualifier/output/report*.json -newermt "$start_time") -# shellcheck disable=SC2068 -for REPORT in ${reports_generated[@]}; do - successful=$(python build/dev/extract_json_field.py report.*.successful "$REPORT") - if echo "${successful}" | grep -iqF true; then - echo "Full success indicated by $REPORT" - else - echo "Could not establish that all uss_qualifier tests passed in $REPORT" - exit 1 - fi -done diff --git a/schemas/monitoring/uss_qualifier/configurations/configuration/ArtifactsConfiguration.json b/schemas/monitoring/uss_qualifier/configurations/configuration/ArtifactsConfiguration.json index 65a548b622..b076160be7 100644 --- a/schemas/monitoring/uss_qualifier/configurations/configuration/ArtifactsConfiguration.json +++ b/schemas/monitoring/uss_qualifier/configurations/configuration/ArtifactsConfiguration.json @@ -7,34 +7,23 @@ "description": "Path to content that replaces the $ref", "type": "string" }, - "graph": { - "description": "If specified, configuration describing a desired graph visualization summarizing the test run", - "oneOf": [ - { - "type": "null" - }, - { - "$ref": "GraphConfiguration.json" - } - ] - }, - "redact_access_tokens": { - "description": "When True, look for instances of \"Authorization\" keys in the report with values starting \"Bearer \" and redact the signature from those access tokens", - "type": "boolean" + "output_path": { + "description": "Path to folder where artifacts should be written.", + "type": "string" }, - "report": { - "description": "Configuration for report generation", + "raw_report": { + "description": "Configuration for raw report generation", "oneOf": [ { "type": "null" }, { - "$ref": "ReportConfiguration.json" + "$ref": "RawReportConfiguration.json" } ] }, "report_html": { - "description": "If specified, configuration describing how an HTML version of the report should be generated", + "description": "If specified, configuration describing how an HTML version of the raw report should be generated", "oneOf": [ { "type": "null" @@ -60,30 +49,24 @@ "items": { "$ref": "TemplatedReportConfiguration.json" }, - "type": "array" - }, - "tested_requirements": { - "description": "If specified, configuration describing a desired report summarizing all tested requirements for each participant", - "oneOf": [ - { - "type": "null" - }, - { - "$ref": "TestedRequirementsConfiguration.json" - } + "type": [ + "array", + "null" ] }, - "tested_roles": { - "description": "If specified, configuration describing a desired report summarizing tested requirements for each specified participant and role", - "oneOf": [ - { - "type": "null" - }, - { - "$ref": "TestedRolesConfiguration.json" - } + "tested_requirements": { + "description": "If specified, list of configurations describing desired reports summarizing tested requirements for each participant", + "items": { + "$ref": "TestedRequirementsConfiguration.json" + }, + "type": [ + "array", + "null" ] } }, + "required": [ + "output_path" + ], "type": "object" } \ No newline at end of file diff --git a/schemas/monitoring/uss_qualifier/configurations/configuration/GraphConfiguration.json b/schemas/monitoring/uss_qualifier/configurations/configuration/GraphConfiguration.json deleted file mode 100644 index 715d83454a..0000000000 --- a/schemas/monitoring/uss_qualifier/configurations/configuration/GraphConfiguration.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "$id": "https://github.com/interuss/monitoring/blob/main/schemas/monitoring/uss_qualifier/configurations/configuration/GraphConfiguration.json", - "$schema": "https://json-schema.org/draft/2020-12/schema", - "description": "monitoring.uss_qualifier.configurations.configuration.GraphConfiguration, as defined in monitoring/uss_qualifier/configurations/configuration.py", - "properties": { - "$ref": { - "description": "Path to content that replaces the $ref", - "type": "string" - }, - "gv_path": { - "description": "Path of GraphViz (.gv) text file to contain a visualization of the test run", - "type": "string" - } - }, - "required": [ - "gv_path" - ], - "type": "object" -} \ No newline at end of file diff --git a/schemas/monitoring/uss_qualifier/configurations/configuration/RawReportConfiguration.json b/schemas/monitoring/uss_qualifier/configurations/configuration/RawReportConfiguration.json new file mode 100644 index 0000000000..860f02ef52 --- /dev/null +++ b/schemas/monitoring/uss_qualifier/configurations/configuration/RawReportConfiguration.json @@ -0,0 +1,23 @@ +{ + "$id": "https://github.com/interuss/monitoring/blob/main/schemas/monitoring/uss_qualifier/configurations/configuration/RawReportConfiguration.json", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "description": "monitoring.uss_qualifier.configurations.configuration.RawReportConfiguration, as defined in monitoring/uss_qualifier/configurations/configuration.py", + "properties": { + "$ref": { + "description": "Path to content that replaces the $ref", + "type": "string" + }, + "indent": { + "description": "To pretty-print JSON content, specify an indent level (generally 2), or omit or set to None to write compactly.", + "type": [ + "integer", + "null" + ] + }, + "redact_access_tokens": { + "description": "When True, look for instances of \"Authorization\" keys in the report with values starting \"Bearer \" and redact the signature from those access tokens", + "type": "boolean" + } + }, + "type": "object" +} \ No newline at end of file diff --git a/schemas/monitoring/uss_qualifier/configurations/configuration/ReportConfiguration.json b/schemas/monitoring/uss_qualifier/configurations/configuration/ReportConfiguration.json deleted file mode 100644 index ea0cff118b..0000000000 --- a/schemas/monitoring/uss_qualifier/configurations/configuration/ReportConfiguration.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "$id": "https://github.com/interuss/monitoring/blob/main/schemas/monitoring/uss_qualifier/configurations/configuration/ReportConfiguration.json", - "$schema": "https://json-schema.org/draft/2020-12/schema", - "description": "monitoring.uss_qualifier.configurations.configuration.ReportConfiguration, as defined in monitoring/uss_qualifier/configurations/configuration.py", - "properties": { - "$ref": { - "description": "Path to content that replaces the $ref", - "type": "string" - }, - "report_path": { - "description": "File name of the report to write (if test_config provided) or read (if test_config not provided)", - "type": "string" - } - }, - "required": [ - "report_path" - ], - "type": "object" -} \ No newline at end of file diff --git a/schemas/monitoring/uss_qualifier/configurations/configuration/ReportHTMLConfiguration.json b/schemas/monitoring/uss_qualifier/configurations/configuration/ReportHTMLConfiguration.json index 1877035482..a836ca7d1d 100644 --- a/schemas/monitoring/uss_qualifier/configurations/configuration/ReportHTMLConfiguration.json +++ b/schemas/monitoring/uss_qualifier/configurations/configuration/ReportHTMLConfiguration.json @@ -7,13 +7,10 @@ "description": "Path to content that replaces the $ref", "type": "string" }, - "html_path": { - "description": "Path of HTML file to contain an HTML rendering of the raw test report object", - "type": "string" + "redact_access_tokens": { + "description": "When True, look for instances of \"Authorization\" keys in the report with values starting \"Bearer \" and redact the signature from those access tokens", + "type": "boolean" } }, - "required": [ - "html_path" - ], "type": "object" } \ No newline at end of file diff --git a/schemas/monitoring/uss_qualifier/configurations/configuration/SequenceViewConfiguration.json b/schemas/monitoring/uss_qualifier/configurations/configuration/SequenceViewConfiguration.json index f1428d02a2..844cf2a771 100644 --- a/schemas/monitoring/uss_qualifier/configurations/configuration/SequenceViewConfiguration.json +++ b/schemas/monitoring/uss_qualifier/configurations/configuration/SequenceViewConfiguration.json @@ -7,13 +7,10 @@ "description": "Path to content that replaces the $ref", "type": "string" }, - "output_path": { - "description": "Path of a folder into which report HTML files should be written", - "type": "string" + "redact_access_tokens": { + "description": "When True, look for instances of \"Authorization\" keys in the report with values starting \"Bearer \" and redact the signature from those access tokens", + "type": "boolean" } }, - "required": [ - "output_path" - ], "type": "object" } \ No newline at end of file diff --git a/schemas/monitoring/uss_qualifier/configurations/configuration/TemplatedReportConfiguration.json b/schemas/monitoring/uss_qualifier/configurations/configuration/TemplatedReportConfiguration.json index 6e26232e54..3bc01fb371 100644 --- a/schemas/monitoring/uss_qualifier/configurations/configuration/TemplatedReportConfiguration.json +++ b/schemas/monitoring/uss_qualifier/configurations/configuration/TemplatedReportConfiguration.json @@ -18,8 +18,8 @@ } ] }, - "output_path": { - "description": "Path of HTML file to contain the rendered templated report", + "report_name": { + "description": "Name of HTML file (without extension) to contain the rendered templated report", "type": "string" }, "template_url": { @@ -28,7 +28,7 @@ } }, "required": [ - "output_path", + "report_name", "template_url" ], "type": "object" diff --git a/schemas/monitoring/uss_qualifier/configurations/configuration/TestedRequirementsConfiguration.json b/schemas/monitoring/uss_qualifier/configurations/configuration/TestedRequirementsConfiguration.json index 35ffb9dadc..15dec41d6f 100644 --- a/schemas/monitoring/uss_qualifier/configurations/configuration/TestedRequirementsConfiguration.json +++ b/schemas/monitoring/uss_qualifier/configurations/configuration/TestedRequirementsConfiguration.json @@ -7,10 +7,6 @@ "description": "Path to content that replaces the $ref", "type": "string" }, - "output_path": { - "description": "Path of a folder into which report HTML files should be written", - "type": "string" - }, "participant_requirements": { "additionalProperties": { "type": "string" @@ -26,6 +22,10 @@ "null" ] }, + "report_name": { + "description": "Name of subfolder in output path to contain the rendered templated report", + "type": "string" + }, "requirement_collections": { "additionalProperties": { "$ref": "../../requirements/definitions/RequirementCollection.json" @@ -43,7 +43,7 @@ } }, "required": [ - "output_path" + "report_name" ], "type": "object" } \ No newline at end of file diff --git a/schemas/monitoring/uss_qualifier/configurations/configuration/TestedRolesConfiguration.json b/schemas/monitoring/uss_qualifier/configurations/configuration/TestedRolesConfiguration.json deleted file mode 100644 index af8cf7a590..0000000000 --- a/schemas/monitoring/uss_qualifier/configurations/configuration/TestedRolesConfiguration.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "$id": "https://github.com/interuss/monitoring/blob/main/schemas/monitoring/uss_qualifier/configurations/configuration/TestedRolesConfiguration.json", - "$schema": "https://json-schema.org/draft/2020-12/schema", - "description": "monitoring.uss_qualifier.configurations.configuration.TestedRolesConfiguration, as defined in monitoring/uss_qualifier/configurations/configuration.py", - "properties": { - "$ref": { - "description": "Path to content that replaces the $ref", - "type": "string" - }, - "report_path": { - "description": "Path of folder to write HTML files containing a fulfilled-requirements-based view of the test report", - "type": "string" - } - }, - "required": [ - "report_path" - ], - "type": "object" -} \ No newline at end of file