[uss_qualifier] Clean up artifacts (#282)

* Clean up artifacts

* Fix dss_probing artifacts

* Fix shell lint

github_pages/make_site_content.sh

@@ -11,6 +11,4 @@ mkdir ./public
 cp -r ./monitoring/github_pages/static/* ./public
 
 mkdir -p ./public/artifacts/uss_qualifier/reports
-cp -r ./artifacts/uss_qualifier/output/sequence_uspace ./public/artifacts/uss_qualifier/reports/sequence_uspace
-cp -r ./artifacts/uss_qualifier/output/tested_requirements_uspace ./public/artifacts/uss_qualifier/reports/tested_requirements_uspace
-cp -r ./artifacts/uss_qualifier/output/capabilities_uspace.html ./public/artifacts/uss_qualifier/reports/capabilities_uspace.html
+cp -r ./artifacts/uss_qualifier/output ./public/artifacts/uss_qualifier/reports

github_pages/static/index.md

@@ -4,10 +4,32 @@ This site contains content automatically generated by actions in the [InterUSS](
 
 ## uss_qualifier [reports](https://github.com/interuss/monitoring/tree/main/monitoring/uss_qualifier/reports)
 
-These reports were generated during continuous integration for the most recent PR merged to the main branch.
+These reports were generated during continuous integration for the most recent PR merged to the main branch.  The test configurations producing this output are in [monitoring/uss_qualifier/configurations/dev](https://github.com/interuss/monitoring/tree/main/monitoring/uss_qualifier/configurations/dev).
 
-### [U-space developer](https://github.com/interuss/monitoring/blob/main/monitoring/uss_qualifier/configurations/dev/uspace.yaml) [test configuration](https://github.com/interuss/monitoring/tree/main/monitoring/uss_qualifier/configurations)
+### [U-space test configuration](https://github.com/interuss/monitoring/blob/main/monitoring/uss_qualifier/configurations/dev/uspace.yaml)
 
-* [Sequence view](./artifacts/uss_qualifier/reports/sequence_uspace)
-* [Tested requirements](./artifacts/uss_qualifier/reports/tested_requirements_uspace)
-* [Demonstrated capabilities](./artifacts/uss_qualifier/reports/capabilities_uspace.html)
+* [Sequence view](./artifacts/uss_qualifier/reports/uspace/sequence)
+* [Tested requirements](./artifacts/uss_qualifier/reports/uspace/requirements)
+* [Demonstrated capabilities](./artifacts/uss_qualifier/reports/uspace/capabilities.html)
+* [Raw report](./artifacts/uss_qualifier/reports/uspace/report.json) (large)
+
+### [No-op test configuration](https://github.com/interuss/monitoring/blob/main/monitoring/uss_qualifier/configurations/dev/noop.yaml)
+
+* [Raw report](./artifacts/uss_qualifier/reports/noop/report.json) (indented to be human-readable)
+* [Interactive report](./artifacts/uss_qualifier/reports/noop/report.html)
+* [Sequence view](./artifacts/uss_qualifier/reports/noop/sequence)
+
+### [ASTM F3548-21 test configuration](https://github.com/interuss/monitoring/blob/main/monitoring/uss_qualifier/configurations/dev/f3548_self_contained.yaml)
+
+* [Sequence view](./artifacts/uss_qualifier/reports/f3548/sequence)
+* [Tested requirements](./artifacts/uss_qualifier/reports/f3548/requirements)
+
+### [ASTM F3411-22a test configuration](https://github.com/interuss/monitoring/blob/main/monitoring/uss_qualifier/configurations/dev/netrid_v22a.yaml)
+
+* [Sequence view](./artifacts/uss_qualifier/reports/netrid_v22a/sequence)
+* [Tested requirements](./artifacts/uss_qualifier/reports/netrid_v22a/requirements)
+
+### [DSS integration test configuration](https://github.com/interuss/monitoring/blob/main/monitoring/uss_qualifier/configurations/dev/dss_probing.yaml)
+
+* [Sequence view](./artifacts/uss_qualifier/reports/dss_probing/sequence)
+* [Tested requirements](./artifacts/uss_qualifier/reports/dss_probing/requirements)

monitoring/uss_qualifier/configurations/configuration.py

@@ -27,18 +27,13 @@ class TestConfiguration(ImplicitDict):
     """Declarations for resources used by the test suite"""
 
 
-class TestedRolesConfiguration(ImplicitDict):
-    report_path: str
-    """Path of folder to write HTML files containing a fulfilled-requirements-based view of the test report"""
-
-
 TestedRequirementsCollectionIdentifier = str
 """Identifier for a requirements collection, local to a TestedRequirementsConfiguration artifact configuration."""
 
 
 class TestedRequirementsConfiguration(ImplicitDict):
-    output_path: str
-    """Path of a folder into which report HTML files should be written"""
+    report_name: str
+    """Name of subfolder in output path to contain the rendered templated report"""
 
     requirement_collections: Optional[
         Dict[TestedRequirementsCollectionIdentifier, RequirementCollection]
@@ -52,13 +47,13 @@ class TestedRequirementsConfiguration(ImplicitDict):
 
 
 class SequenceViewConfiguration(ImplicitDict):
-    output_path: str
-    """Path of a folder into which report HTML files should be written"""
+    redact_access_tokens: bool = True
+    """When True, look for instances of "Authorization" keys in the report with values starting "Bearer " and redact the signature from those access tokens"""
 
 
 class ReportHTMLConfiguration(ImplicitDict):
-    html_path: str
-    """Path of HTML file to contain an HTML rendering of the raw test report object"""
+    redact_access_tokens: bool = True
+    """When True, look for instances of "Authorization" keys in the report with values starting "Bearer " and redact the signature from those access tokens"""
 
 
 class TemplatedReportInjectedConfiguration(ImplicitDict):
@@ -69,44 +64,36 @@ class TemplatedReportConfiguration(ImplicitDict):
     template_url: str
     """Url of the template to download from"""
 
-    output_path: str
-    """Path of HTML file to contain the rendered templated report"""
+    report_name: str
+    """Name of HTML file (without extension) to contain the rendered templated report"""
 
     configuration: Optional[TemplatedReportInjectedConfiguration] = None
     """Configuration to be injected in the templated report"""
 
 
-class GraphConfiguration(ImplicitDict):
-    gv_path: str
-    """Path of GraphViz (.gv) text file to contain a visualization of the test run"""
-
+class RawReportConfiguration(ImplicitDict):
+    redact_access_tokens: bool = True
+    """When True, look for instances of "Authorization" keys in the report with values starting "Bearer " and redact the signature from those access tokens"""
 
-class ReportConfiguration(ImplicitDict):
-    report_path: str
-    """File name of the report to write (if test_config provided) or read (if test_config not provided)"""
+    indent: Optional[int] = None
+    """To pretty-print JSON content, specify an indent level (generally 2), or omit or set to None to write compactly."""
 
 
 class ArtifactsConfiguration(ImplicitDict):
-    redact_access_tokens: bool = True
-    """When True, look for instances of "Authorization" keys in the report with values starting "Bearer " and redact the signature from those access tokens"""
+    output_path: str
+    """Path to folder where artifacts should be written."""
 
-    report: Optional[ReportConfiguration] = None
-    """Configuration for report generation"""
+    raw_report: Optional[RawReportConfiguration] = None
+    """Configuration for raw report generation"""
 
     report_html: Optional[ReportHTMLConfiguration] = None
-    """If specified, configuration describing how an HTML version of the report should be generated"""
+    """If specified, configuration describing how an HTML version of the raw report should be generated"""
 
-    templated_reports: List[TemplatedReportConfiguration] = []
+    templated_reports: Optional[List[TemplatedReportConfiguration]] = None
     """List of report templates to be rendered"""
 
-    graph: Optional[GraphConfiguration] = None
-    """If specified, configuration describing a desired graph visualization summarizing the test run"""
-
-    tested_roles: Optional[TestedRolesConfiguration] = None
-    """If specified, configuration describing a desired report summarizing tested requirements for each specified participant and role"""
-
-    tested_requirements: Optional[TestedRequirementsConfiguration] = None
-    """If specified, configuration describing a desired report summarizing all tested requirements for each participant"""
+    tested_requirements: Optional[List[TestedRequirementsConfiguration]] = None
+    """If specified, list of configurations describing desired reports summarizing tested requirements for each participant"""
 
     sequence_view: Optional[SequenceViewConfiguration] = None
     """If specified, configuration describing a desired report describing the sequence of events that occurred during the test"""

monitoring/uss_qualifier/configurations/dev/dss_probing.yaml

@@ -26,19 +26,20 @@ v1:
           service_area: kentland_service_area
           problematically_big_area: kentland_problematically_big_area
   artifacts:
-    report:
-      report_path: output/report_dss_probing.json
+    output_path: output/dss_probing
+    raw_report: {}
+    sequence_view: {}
     tested_requirements:
-      output_path: output/tested_requirements_dss_probing
-      requirement_collections:
-        all_astm_dss_requirements:
-          requirement_collections:
-            - requirement_sets:
-                - astm.f3411.v22a.dss_provider
-                - astm.f3411.v19.dss_provider
-      participant_requirements:
-        uss1: all_astm_dss_requirements
-        uss2: all_astm_dss_requirements
+      - report_name: requirements
+        requirement_collections:
+          all_astm_dss_requirements:
+            requirement_collections:
+              - requirement_sets:
+                  - astm.f3411.v22a.dss_provider
+                  - astm.f3411.v19.dss_provider
+        participant_requirements:
+          uss1: all_astm_dss_requirements
+          uss2: all_astm_dss_requirements
   validation:
     criteria:
       - full_success: {}

monitoring/uss_qualifier/configurations/dev/f3548_self_contained.yaml

@@ -93,23 +93,23 @@ v1:
   # relative to where uss_qualifier is executed from, and are located inside the
   # Docker container executing uss_qualifier.
   artifacts:
+    # Write artifacts to this folder (relative to uss_qualifier)
+    output_path: output/f3548
+
     # Write out full report content
-    report:
-      # Path to main report output
-      report_path: output/report_f3548_self_contained.json
+    raw_report: {}
 
-    # Write out a human-readable report of the requirements tested
+    # Write out a human-readable report of the F3548-21 requirements tested
     tested_requirements:
-      output_path: output/tested_requirements_f3548_self_contained
-      requirement_collections:
-        scd:
-          requirement_collections:
-            - requirement_sets:
-                - astm.f3548.v21.scd
-      participant_requirements:
-        uss1: scd
-        uss2: scd
+      - report_name: requirements
+        requirement_collections:
+          scd:
+            requirement_collections:
+              - requirement_sets:
+                  - astm.f3548.v21.scd
+        participant_requirements:
+          uss1: scd
+          uss2: scd
 
     # Write out a human-readable report showing the sequence of events of the test
-    sequence_view:
-      output_path: output/sequence_f3548_self_contained
+    sequence_view: {}

monitoring/uss_qualifier/configurations/dev/general_flight_auth.yaml

@@ -10,7 +10,7 @@ v1:
         resources:
           table: example_flight_check_table
   artifacts:
-    report:
-      report_path: output/report_general_flight_auth.json
+    output_path: output/general_flight_auth
+    raw_report: {}
   validation:
     $ref: ./library/validation.yaml#/normal_test

monitoring/uss_qualifier/configurations/dev/geoawareness_cis.yaml

@@ -13,9 +13,7 @@ v1:
         resources:
           source_document: source_document
   artifacts:
-    report:
-      report_path: output/report_geoawareness_cis.json
-    graph:
-      gv_path: output/report_geoawareness_cis.gv
+    output_path: output/geoawareness_cis
+    raw_report: {}
   validation:
     $ref: ./library/validation.yaml#/normal_test

monitoring/uss_qualifier/configurations/dev/geospatial_comprehension.yaml

@@ -10,7 +10,7 @@ v1:
         resources:
           table: example_feature_check_table
   artifacts:
-    report:
-      report_path: output/report_geospatial_comprehension.json
+    output_path: output/geospatial_comprehension
+    raw_report: {}
   validation:
     $ref: ./library/validation.yaml#/normal_test

monitoring/uss_qualifier/configurations/dev/library/resources.yaml

@@ -101,13 +101,13 @@ adjacent_circular_storage_config:
   $content_schema: monitoring/uss_qualifier/resources/definitions/ResourceDeclaration.json
   resource_type: resources.netrid.FlightDataStorageResource
   specification:
-    flight_record_collection_path: "./output/test_data.che.netrid.circular_flights.json"
+    flight_record_collection_path: "./output/generate_rid_test_data/flight_data/test_data.che.netrid.circular_flights.json"
 
 kml_storage_config:
   $content_schema: monitoring/uss_qualifier/resources/definitions/ResourceDeclaration.json
   resource_type: resources.netrid.FlightDataStorageResource
   specification:
-    flight_record_collection_path: "./output/test_data.usa.netrid.dcdemo_flights.json"
+    flight_record_collection_path: "./output/generate_rid_test_data/flight_data/test_data.usa.netrid.dcdemo_flights.json"
 
 # ===== Flight planning intents =====
 

monitoring/uss_qualifier/configurations/dev/message_signing.yaml

@@ -50,5 +50,5 @@ v1:
           dss: scd_dss
 
   artifacts:
-    report:
-      report_path: output/report_message_signing.json
+    output_path: output/message_signing
+    raw_report: {}

monitoring/uss_qualifier/configurations/dev/netrid_v19.yaml

@@ -31,26 +31,23 @@ v1:
           service_area: kentland_service_area
           problematically_big_area: au_problematically_big_area
   artifacts:
-    report:
-      report_path: output/report_netrid_v19.json
-    tested_roles:
-      report_path: output/tested_roles_netrid_v19
+    output_path: output/netrid_v19
+    raw_report: {}
     tested_requirements:
-      output_path: output/tested_requirements_f3411v19
-      requirement_collections:
-        sp_dp_dss:
-          requirement_sets:
-            - astm.f3411.v19.service_provider#Tested by automated tests
-            - astm.f3411.v19.display_provider#Automated verification
-            - astm.f3411.v19.dss_provider
-        sp_dss:
-          requirement_sets:
-            - astm.f3411.v19.service_provider#Tested by automated tests
-            - astm.f3411.v19.dss_provider
-      participant_requirements:
-        uss1: sp_dp_dss
-        uss2: sp_dss
-    sequence_view:
-      output_path: output/sequence_netrid_v19
+      - report_name: requirements
+        requirement_collections:
+          sp_dp_dss:
+            requirement_sets:
+              - astm.f3411.v19.service_provider#Tested by automated tests
+              - astm.f3411.v19.display_provider#Automated verification
+              - astm.f3411.v19.dss_provider
+          sp_dss:
+            requirement_sets:
+              - astm.f3411.v19.service_provider#Tested by automated tests
+              - astm.f3411.v19.dss_provider
+        participant_requirements:
+          uss1: sp_dp_dss
+          uss2: sp_dss
+    sequence_view: {}
   validation:
     $ref: ./library/validation.yaml#/normal_test

monitoring/uss_qualifier/configurations/dev/netrid_v22a.yaml

@@ -31,26 +31,23 @@ v1:
           service_area: kentland_service_area
           problematically_big_area: au_problematically_big_area
   artifacts:
-    report:
-      report_path: output/report_netrid_v22a.json
-    tested_roles:
-      report_path: output/tested_roles_netrid_v22a
+    output_path: output/netrid_v22a
+    raw_report: {}
     tested_requirements:
-      output_path: output/tested_requirements_f3411v22a
-      requirement_collections:
-        sp_dp_dss:
-          requirement_sets:
-            - astm.f3411.v22a.service_provider#Mandatory requirements
-            - astm.f3411.v22a.display_provider#Mandatory requirements
-            - astm.f3411.v22a.dss_provider
-        sp_dss:
-          requirement_sets:
-            - astm.f3411.v22a.service_provider#Mandatory requirements
-            - astm.f3411.v22a.dss_provider
-      participant_requirements:
-        uss1: sp_dp_dss
-        uss2: sp_dss
-    sequence_view:
-      output_path: output/sequence_netrid_v22a
+      - report_name: requirements
+        requirement_collections:
+          sp_dp_dss:
+            requirement_sets:
+              - astm.f3411.v22a.service_provider#Mandatory requirements
+              - astm.f3411.v22a.display_provider#Mandatory requirements
+              - astm.f3411.v22a.dss_provider
+          sp_dss:
+            requirement_sets:
+              - astm.f3411.v22a.service_provider#Mandatory requirements
+              - astm.f3411.v22a.dss_provider
+        participant_requirements:
+          uss1: sp_dp_dss
+          uss2: sp_dss
+    sequence_view: {}
   validation:
     $ref: ./library/validation.yaml#/normal_test

monitoring/uss_qualifier/configurations/dev/noop.yaml

@@ -24,8 +24,10 @@ v1:
           noop_config: noop_config
 
   artifacts:
-    report:
-      # Path to main report output
-      report_path: output/report_noop.json
+    output_path: output/noop
+    raw_report:
+      indent: 2
+    sequence_view: {}
+    report_html: {}
   validation:
     $ref: ./library/validation.yaml#/normal_test