Add Golden Copy Tests

.github/workflows/automated-staging-test-run.yml

@@ -6,23 +6,9 @@ on:
   workflow_dispatch:
 
 jobs:
-  test_files:
-    runs-on: ubuntu-latest
-
-    steps:
-      - name: Check out the repository
-        uses: actions/checkout@v4
-
-      - name: Run automated tests
-        env:
-          AZURE_STORAGE_CONNECTION_STRING: ${{ secrets.AUTOMATED_TEST_AZURE_STORAGE_CONNECTION_STRING }}
-        run: ./gradlew rs-e2e:clean rs-e2e:automatedTest
-
-      - name: Send slack notification on test failure
-        if: failure()
-        uses: slackapi/[email protected]
-        with:
-          webhook: ${{ secrets.SLACK_WEBHOOK_URL }}
-          webhook-type: incoming-webhook
-          payload: |
-            text: "Automated Staging RS Integration Test Failed!: https://github.com/CDCgov/trusted-intermediary/actions/runs/${{ github.run_id }}"
+  call-staging-test-run:
+    uses: ./.github/workflows/staging-test-run_reusable.yml
+    with:
+      NAME: 'Automated'
+      TEST_TYPE: 'test defined assertions on relevant messages'
+    secrets: inherit # pragma: allowlist secret

.github/workflows/automated-staging-test-submit.yml

@@ -18,8 +18,8 @@ jobs:
 
       - name: Install jwt-cli
         run: |
-            curl --silent --location https://github.com/mike-engel/jwt-cli/releases/latest/download/jwt-linux.tar.gz | tar xvz -C /usr/local/bin/
-            sudo chmod +x /usr/local/bin/jwt
+          curl --silent --location https://github.com/mike-engel/jwt-cli/releases/latest/download/jwt-linux.tar.gz | tar xvz -C /usr/local/bin/
+          sudo chmod +x /usr/local/bin/jwt
 
       - name: Write private key to file
         run: |
@@ -28,32 +28,55 @@ jobs:
 
       - name: Send HL7 sample messages to staging RS
         run: |
-            host=https://staging.prime.cdc.gov:443
-            client_id=flexion
-            client_sender=simulated-sender
-            jwt=$(jwt encode --exp='+5min' --jti $(uuidgen) --alg RS256 \
-                -k $client_id.$client_sender -i $client_id.$client_sender \
-                -s $client_id.$client_sender -a $host --no-iat -S @/tmp/staging_private_key.pem)
-
-            token=$(curl \
-                    --header "Content-Type: application/x-www-form-urlencoded" \
-                    --data "scope=$client_id.*.report" \
-                    --data "client_assertion=$jwt" \
-                    --data "client_assertion_type=urn:ietf:params:oauth:client-assertion-type:jwt-bearer" \
-                    --data "grant_type=client_credentials" \
-                    --silent \
-                    "$host/api/token" | jq -r ".access_token")
-
-            for file in $(pwd)/examples/Test/Automated/*.hl7; do
-                echo "Sending $file"
-                curl \
-                    --header "Content-Type: application/hl7-v2" \
-                    --header "Client: $client_id.$client_sender" \
-                    --header "Authorization: Bearer $token" \
-                    --data-binary "@$file" \
-                    --silent \
-                    "$host/api/waters"
-            done
+          shopt -s globstar
+          host=https://staging.prime.cdc.gov:443
+          client_id=flexion
+          client_sender=simulated-sender
+          jwt=$(jwt encode --exp='+5min' --jti $(uuidgen) --alg RS256 \
+              -k $client_id.$client_sender -i $client_id.$client_sender \
+              -s $client_id.$client_sender -a $host --no-iat -S @/tmp/staging_private_key.pem)
+
+          token=$(curl \
+                  --header "Content-Type: application/x-www-form-urlencoded" \
+                  --data "scope=$client_id.*.report" \
+                  --data "client_assertion=$jwt" \
+                  --data "client_assertion_type=urn:ietf:params:oauth:client-assertion-type:jwt-bearer" \
+                  --data "grant_type=client_credentials" \
+                  --silent \
+                  "$host/api/token" | jq -r ".access_token")
+
+          failed_to_send=""
+          for file in $(pwd)/examples/Test/Automated/{Assertion,GoldenCopy/Input}/*.hl7; do
+              echo "Sending $file"
+              response=$(curl \
+                  --header "Content-Type: application/hl7-v2" \
+                  --header "Client: $client_id.$client_sender" \
+                  --header "Authorization: Bearer $token" \
+                  --data-binary "@$file" \
+                  --silent \
+                  "$host/api/waters" | jq -r ".overallStatus")
+
+              echo "Response to curl: ${response}"
+
+              if [ "$response" != "Received" ]; then
+                failed_to_send+="${file}"$'\n'
+              fi
+          done
+
+          if [ "$failed_to_send" ]; then
+            echo "Issue sending files:"$'\n'"${failed_to_send}"
+            exit 1
+          fi
+
+      - name: Send slack notification on test failure
+        if: failure()
+        uses: slackapi/[email protected]
+        with:
+          webhook: ${{ secrets.SLACK_WEBHOOK_URL }}
+          webhook-type: incoming-webhook
+          payload: |
+            text: "Submit Staging RS Integration Test Failed!: https://github.com/CDCgov/trusted-intermediary/actions/runs/${{ github.run_id }}"
+
 
       - name: Clean up private key
         if: always()

.github/workflows/golden-copy-staging-test-run.yml

@@ -0,0 +1,15 @@
+name: Golden Copy Staging Test - Run integration tests
+
+on:
+  pull_request:
+  schedule:
+    - cron: "10 7 * * 2-6"  # Tuesday to Saturday at 2am EST (7:10am UTC) - two hours after `automated-staging-test-submit` runs
+  workflow_dispatch:
+
+jobs:
+  call-staging-test-run:
+    uses: ./.github/workflows/staging-test-run_reusable.yml
+    with:
+      NAME: 'Golden Copy'
+      TEST_TYPE: 'test golden copy files on actual files'
+    secrets: inherit # pragma: allowlist secret

.github/workflows/staging-test-run_reusable.yml

@@ -0,0 +1,43 @@
+name: Staging Test - Run integration tests
+
+on:
+  workflow_call:
+    inputs:
+      TEST_TYPE:
+        type: string
+        description: The e2e test to run
+        required: true
+        default: 'automatedTest'
+      NAME:
+        type: string
+        description: The name of test
+        required: true
+        default: 'Automated'
+    secrets:
+        AUTOMATED_TEST_AZURE_STORAGE_CONNECTION_STRING:
+          required: true
+        SLACK_WEBHOOK_URL:
+          required: true
+
+jobs:
+  test_files:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Check out the repository
+        uses: actions/checkout@v4
+
+      - name: Run automated tests
+        env:
+          AZURE_STORAGE_CONNECTION_STRING: ${{ secrets.AUTOMATED_TEST_AZURE_STORAGE_CONNECTION_STRING }}
+        run: |
+          ./gradlew rs-e2e:clean rs-e2e:automatedTest --tests="gov.hhs.cdc.trustedintermediary.rse2e.AutomatedTest.${{inputs.TEST_TYPE}}"
+
+      - name: Send slack notification on test failure
+        if: failure()
+        uses: slackapi/[email protected]
+        with:
+          webhook: ${{ secrets.SLACK_WEBHOOK_URL }}
+          webhook-type: incoming-webhook
+          payload: |
+            text: "${{ inputs.NAME }} Staging RS Integration Test Failed!: https://github.com/CDCgov/trusted-intermediary/actions/runs/${{ github.run_id }}"

adr/025-automated-rs-integration-test.md

@@ -9,7 +9,7 @@ Date: 2024-10-02
 2. We decided to create Github Action workflows to schedule two tasks:
    - One workflow submits sample HL7 files to ReportStream in staging, with output files expected to be delivered to an Azure blob container
    - Another workflow later runs the integration tests on the output and input files
-3. We decided to use MSH-10 to match the input and output files, and to filter the receivers in ReportStream when MSH-6.2 not available.
+3. We decided to use MSH-10 to match the input and output files, and to filter the receivers in ReportStream when MSH-6.2 not available, and MSH-3 to distinguish between our golden copy and assertion tests.
 
 ## Status
 
@@ -40,7 +40,7 @@ built in extra time in case of any issues that cause delays.
 
 ### Decision 3
 
-We're using the value in MSH-10 for two purposes: matching input and output files, and some filtering in RS.
+We're using the value in MSH-10 for two purposes: matching input and output files, and some filtering in RS. We use MSH-3 for a similar reason, specifically for filtering and renaming the file to include a flag.
 
 We chose MSH-10 to match files on because it's a value that shouldn't change and should be unique to
 a particular message. We're also using it to route these test messages because in some cases, we apply

examples/README.md

@@ -21,8 +21,8 @@
 ## Routing of these files
 
 To avoid routing issues, we have decided to use `MSH-11` for routing of test messages ([more context here](/adr/026-hl7-test-message-routing.md)). The values we have decided to use are:
-- `D`: for test files **not** to be sent to partners and to be sent manually. Any files under `examples/` and not in `examples/Test/Automated/` should have this value
-- `N`: for test files **not** to be sent to partners and sent by a scheduled task. Any files under `examples/Test/Automated/` should have this value
+- `D`: for test files **not** to be sent to partners and to be sent manually. Any files under `examples/` and not in `examples/Test/Automated/Assertion/` should have this value
+- `N`: for test files **not** to be sent to partners and sent by a scheduled task. Any files under `examples/Test/Automated/Assertion/` should have this value
 - `T`: for test files to be sent manually to partners. `P` will also be routed to partners
 
 **Note**: for some sample files, our transformations **will** rewrite the `MSH-5` and/or `MSH-6` HL7 fields normally used for routing, so we can't rely only on those fields to route. This is the case for most of the files in the `examples/CA` folder. If you are sending any files in that folder and you don't want the message to be delivered to our partner, please make sure `MSH-11` is **not** `T` or `P`. Otherwise the message will be delivered to our partner regardless of what is there in `MSH-5` and `MSH-6`. Please see [this ADR](/adr/026-hl7-test-message-routing.md) for more context
@@ -38,7 +38,7 @@ In order to keep the snapshots up-to-date, we have a script that automates the r
 - The `ORM` messages with ids `003`, `004`, `005`, `006`, `007`, `008`, `009`, `010` in the `Test/Orders` folder were modified to comply with current requirements for ReportStream, as it doesn't yet support HL7 `2.3`:
   - Added `MSH-9.3`
   - Changed `MSH-10` to `2.5.1`
-- The `MSH-11` value for all sample files in `examples/` (with the exception of files in `examples/Test/Automated/`) was changed to `D`. This is to comply with our routing filters in RS for test messages
+- The `MSH-11` value for all sample files in `examples/` (with the exception of files in `examples/Test/Automated/Assertion/`) was changed to `D`. This is to comply with our routing filters in RS for test messages
 
 ## Previously renamed files
 

examples/Test/Automated/GoldenCopy/Expected/008_CA_ORU_R01_CDPH_UCSD_MRN_80015955_pre_TI_TC007B_expected.hl7

@@ -0,0 +1 @@
+MSH|^~\&|GOLDEN-COPY|CDPH|EPIC|R797|20241206093739||ORU^R01|008|N|2.5.1PID|1||80015955||INTERFACE^BOY A CDPHONE||202411180830|M||2131-1^Other Race||||||||||||UNK^Unknown||Y|1NK1|1|INTERFACE^CDPHONE|MTH^Mother|456 ST^^SAN DIEGO^CA^92123^USA|^^^^^858^2490016|||||||||||19921118ORC|RE|^HospOrdNumber||7902698491^FormNumber||||||||1427255868^JOHNSON^KATHRYN^^^^^^^^^^NPI|||||||||^^^^^^^^^R797|AG N 005EVA DLEIFR, F N 0037TERTS ONSER^^SAN DIEGO^CA^99999-9999OBR|1|^HospOrdNumber||54089-8^NB Screen Panel Patient AHIC^CDPHGSPEAP|||202411021023|||||||202411031023||1427255868^JOHNSON^KATHRYN^^^^^^^^^^NPI||||||20241206093739|||FOBX|1|TX|57724-7^Newborn screening short narrative summary^LN|1|ACTION REQUIRED\.br\\.br\Testing Lab - Quest \R\ S W 0094DVLB TESNU, SAN JUAN CAPISTRANO, CA 99999-9999\.br\\.br\Lab Director - Daisydoublek Toulousep, MD,PhD, Quest, (491)154-5445\.br\\.br\SCID Reviewer - Meadowsaffronpa Maxrm\.br\\.br\Lab Remarks: CD3/CD8/CD45RO, Percent:This test was developed and its analytical performance characteristics have been determined by Quest Diagnostics Nichols Institute San Juan Capistrano. It has not been cleared or approved by FDA. This assay has been validated pursuant to the CLIA regulations and is used for clinical purposes. \.br\\.br\Follow-up:\.br\\.br\Clinical Reviewer Comments: Testing\.br\\.br\Follow-up Recommendation: Refer to CCS-approved Immunology Center\.br\\.br\Disclaimer:\.br\\.br\This test was developed and its performance characteristics have been determined by Quest Diagnostics Nichols Institute, San Juan Capistrano. It has not been cleared or approved by the U.S. Food and Drug Administration. The FDA has determined that such clearance or approval is not necessary. Performance characteristics refer to the analytical performance of the test. \.br\\.br\|||N|||F|||20241206093739OBX|2|TM|57715-5^Time of birth^LN|1|0830|||N|||F|||20241206093739OBX|3|NM|73806-2^Newborn age in hours^LN|1|73|hour(s)||N|||F|||20241206093739OBX|4|TX|62327-2^Post-discharge provider practice address^LN|1|AG N 005EVA DLEIFR, F N 0037TERTS ONSER SAN DIEGO CA 99999-9999 USA|||N|||F|||20241206093739OBX|5|NM|6690-2^Leukocytes [#/volume] in Blood by Automated count^LN|1|45.1|Thousand/uL|9.0-30.0|H|||F|||20241206093739OBX|6|NM|789-8^Erythrocytes [#/volume] in Blood by Automated count^LN|1|4.25|Million/uL|3.90-5.90||||F|||20241206093739OBX|7|NM|718-7^Hemoglobin [Mass/volume] in Blood^LN|1|18.1|g/dL|13.4-19.9||||F|||20241206093739OBX|8|NM|4544-3^Hematocrit [Volume Fraction] of Blood by Automated count^LN|1|50.1|%|42.0-65.0||||F|||20241206093739OBX|9|NM|787-2^Erythrocyte mean corpuscular volume [Entitic volume] by Automated count^LN|1|100.0|fL|88.0-123.0||||F|||20241206093739OBX|10|NM|785-6^Erythrocyte mean corpuscular hemoglobin [Entitic mass] by Automated count^LN|1|35.1|pg|31.0-37.0||||F|||20241206093739OBX|11|NM|786-4^Erythrocyte mean corpuscular hemoglobin concentration [Mass/volume] by Automated count^LN|1|30.2|g/dL|28.0-36.0||||F|||20241206093739OBX|12|NM|777-3^Platelets [#/volume] in Blood by Automated count^LN|1|200|Thousand/uL|150-450||||F|||20241206093739OBX|13|NM|776-5^Platelet mean volume [Entitic volume] in Blood by Rees-Ecker^LN|1|11.1|fL|7.5-12.5||||F|||20241206093739OBX|14|NM|788-0^Erythrocyte distribution width [Ratio] by Automated count^LN|1|14.1|%|13.0-18.0||||F|||20241206093739OBX|15|NM|751-8^Neutrophils [#/volume] in Blood by Automated count^LN|1|7000|cells/uL|5000-21000||||F|||20241206093739OBX|16|NM|26507-4^Neutrophils.band form [#/volume] in Blood^LN^29153|1|3000|cells/uL|1500-4200||||F|||20241206093739OBX|17|NM|30433-7^Metamyelocytes [#/volume] in Blood^LN^29154|1|1|cells/uL|0|H|||F|||20241206093739OBX|18|NM|30446-9^Myelocytes [#/volume] in Blood^LN^29162|1|0|cells/uL|0||||F|||20241206093739OBX|19|NM|26523-1^Promyelocytes [#/volume] in Blood^LN^29163|1|0|cells/uL|0||||F|||20241206093739OBX|20|NM|731-0^Lymphocytes [#/volume] in Blood by Automated count^LN^29164|1|6000|cells/uL|2000-11500||||F|||20241206093739OBX|21|NM|742-7^Monocytes [#/volume] in Blood by Automated count^LN^29165|1|1200|cells/uL|300-2400||||F|||20241206093739OBX|22|NM|711-2^Eosinophils [#/volume] in Blood by Automated count^LN^29166|1|600|cells/uL|15-800||||F|||20241206093739OBX|23|NM|704-7^Basophils [#/volume] in Blood by Automated count^LN^29167|1|200|cells/uL|0-300||||F|||20241206093739OBX|24|NM|30376-8^Blasts [#/volume] in Blood^LN^29168|1|0|cells/uL|0||||F|||20241206093739OBX|25|NM|30392-5^Nucleated erythrocytes [#/volume] in Blood^LN^29169|1|300|cells/uL|0-5000||||F|||20241206093739OBX|26|NM|770-8^Neutrophils/100 leukocytes in Blood by Automated count^LN^29170|1|20|%|||||F|||20241206093739OBX|27|NM|26508-2^Neutrophils.band form/100 leukocytes in Blood^LN^29171|1|2|%|||||F|||20241206093739OBX|28|NM|28541-1^Metamyelocytes/100 leukocytes in Blood^LN^29172|1|1|%|||||F|||20241206093739OBX|29|NM|26498-6^Myelocytes/100 leukocytes in Blood^LN^29173|1|1|%|||||F|||20241206093739OBX|30|NM|26524-9^Promyelocytes/100 leukocytes in Blood^LN^29174|1|1|%|||||F|||20241206093739OBX|31|NM|736-9^Lymphocytes/100 leukocytes in Blood by Automated count^LN^29175|1|1|%|||||F|||20241206093739OBX|32|NM|13046-8^Lymphocytes Variant/100 leukocytes in Blood^LN^29176|1|1|%|0-10||||F|||20241206093739OBX|33|NM|5905-5^Monocytes/100 leukocytes in Blood by Automated count^LN^29177|1|1|%|||||F|||20241206093739OBX|34|NM|713-8^Eosinophils/100 leukocytes in Blood by Automated count^LN^29178|1|1|%|||||F|||20241206093739OBX|35|NM|706-2^Basophils/100 leukocytes in Blood by Automated count^LN^29179|1|1|%|||||F|||20241206093739OBX|36|NM|26446-5^Blasts/100 leukocytes in Blood^LN^29180|1|1|%|||||F|||20241206093739OBX|37|NM|19048-8^Nucleated erythrocytes/100 leukocytes [Ratio] in Blood^LN^29181|1|1|/100 WBC|0|H|||F|||20241206093739OBX|38|NM|8122-4^CD3 cells [#/volume] in Blood^LN^86007711^CD3+ T-Cells, Absolute^L|1|3000|cells/uL|2500-5500||||F|||20241206093739OBX|39|NM|8124-0^CD3 cells/100 cells in Blood^LN^86007712^CD3+ T-Cells, Percent^L|1|60|% of Lymphs|53-84||||F|||20241206093739OBX|40|NM|24467-3^CD3+CD4+ (T4 helper) cells [#/volume] in Blood^LN^86007713^CD3+/CD4+ T-Helper, Abs^L|1|2000|cells/uL|1600-4000||||F|||20241206093739OBX|41|NM|8123-2^CD3+CD4+ (T4 helper) cells/100 cells in Blood^LN^86007714^CD3+/CD4+ T-Helper,Percent^L|1|40|% of Lymphs|35-64||||F|||20241206093739OBX|42|NM|14135-8^CD3+CD8+ (T8 suppressor cells) cells [#/volume] in Blood^LN^86007715^CD3+/CD8+ T-Cytotoxic, Abs^L|1|1000|cells/uL|560-1700||||F|||20241206093739OBX|43|NM|8101-8^CD3+CD8+ (T8 suppressor cells) cells/100 cells in Blood^LN^86007716^CD3+/CD8+T-Cytotox,Percent^L|1|15|% of Lymphs|12-28||||F|||20241206093739OBX|44|NM|8116-6^CD19 cells [#/volume] in Blood^LN^86007717^CD19+ B-cells, Absolute^L|1|500|cells/uL|300-2000||||F|||20241206093739OBX|45|NM|8117-4^CD19 cells/100 cells in Blood^LN^86007718^CD19+ B-cells, Percent^L|1|10|% of Lymphs|6-32||||F|||20241206093739OBX|46|NM|9728-7^CD16C+CD56+ cells [#/volume] in Blood^LN^86007719^CD16/CD56 NK-cells, Abs^L|1|200|cells/uL|170-1100||||F|||20241206093739OBX|47|NM|8112-5^CD16+CD56+ cells/100 cells in Blood^LN^86007720^CD16/CD56 NK-cells,Percent^L|1|9|% of Lymphs|4-18||||F|||20241206093739OBX|48|NM|26759-1^CD4+CD45RA+cells [#/volume] in Blood^LN^86007721^CD3/CD4/CD45RA, Absolute^L|1|1500|cells/uL|1200-3700||||F|||20241206093739OBX|49|NM|13336-3^CD4+CD45RA+cells/100 cells in Blood^LN^^CD3/CD4/CD45RA, Percent^L|1|8|% CD4|||||F|||20241206093739OBX|50|NM|82743-6^CD8+CD45RA+ Cells # Bld^LN^86007723^CD3/CD8/CD45RA, Absolute^L|1|600|cells/uL|450-1500||||F|||20241206093739OBX|51|NM|82744-4^CD8+CD45RA+ Cells/100 CD8 NFr Bld^LN^86007724^CD3/CD8/CD45RA, Percent^L|1|10|% CD8|||||F|||20241206093739OBX|52|NM|85792-0^CD3+CD4+CD45RO+ # Bld^LN^86007725^CD3/CD4/CD45RO, Absolute^L|1|500|cells/uL|60-900||||F|||20241206093739OBX|53|NM|85791-2^CD3+CD4+CD45RO+/100 CD3 NFr Bld^LN^86007726^CD3/CD4/CD45RO, Percent^L|1|5|% CD4|2-22||||F|||20241206093739OBX|54|NM|85790-4^CD3+CD8+CD45RO+ # Bld^LN^86007727^CD3/CD8/CD45RO, Absolute^L|1|50|cells/uL|30-330||||F|||20241206093739OBX|55|NM|85789-6^CD3+CD8+CD45RO+/100 CD3 NFr Bld^LN^86007728^CD3/CD8/CD45RO, Percent^L|1|2|% CD8|1-9||||F|||20241206093739OBX|56|CE|62321-5^Severe combined immunodeficiency newborn screen interpretation^LN|1|LA18593-6^Out of range^LN|||A|||F|||20241206093739OBX|57|TX|62322-3^Severe combined immunodeficiency newborn screening comment-discussion^LN|1|Refer to CCS-approved Immunology Center|||A|||F|||20241206093739