BFD-3091: Use dynamic versions in ansible (#2093)

.github/workflows/ci-ansible.yml

@@ -5,6 +5,9 @@ on:
       role:
         required: true
         type: string
+      bfd_version:
+        required: true
+        type: string
       image_tag:
         required: false
         type: string
@@ -20,4 +23,4 @@ jobs:
       - name: 'Run Tests for ${{ inputs.role }} Role'
         run: |
           export IMAGE_TAG="${image_tag:-$(git rev-parse --short HEAD)}"
-          ops/ansible/roles/${{ inputs.role }}/test/run-tests.sh "$IMAGE_TAG"
+          ops/ansible/roles/${{ inputs.role }}/test/run-tests.sh -i "$IMAGE_TAG" ${{ inputs.bfd_version }}

.github/workflows/ci-java.yml

@@ -72,6 +72,8 @@ jobs:
   mvn-verify:
     runs-on: ubuntu-20.04
     needs: workflow
+    outputs:
+      BFD_PARENT_VERSION: ${{ steps.bfd-parent-version.outputs.BFD_PARENT_VERSION }}
     if: needs.workflow.outputs.files
     steps:
       - name: 'Checkout repo'
@@ -120,9 +122,14 @@ jobs:
           echo "${{ secrets.GITHUB_TOKEN }}" \
           | docker login ghcr.io -u ${{ github.actor }} --password-stdin
 
+      - name: Determine the BFD Parent Version
+        id: bfd-parent-version
+        run: |
+          echo "BFD_PARENT_VERSION=$(yq '.project.version' apps/pom.xml)" >> "$GITHUB_OUTPUT"
+          echo "BFD_PARENT_VERSION=$(yq '.project.version' apps/pom.xml)" >> "$GITHUB_ENV"
+
       - name: Build and Deliver Apps Container Image
         run: |
-          BFD_PARENT_VERSION="$(yq '.project.version' pom.xml)"
           SANITIZED_REF="${GITHUB_REF_NAME////-}"
           IMAGE_NAME="ghcr.io/cmsgov/bfd-apps"
           SHORT_SHA="$(git rev-parse --short HEAD)"
@@ -199,15 +206,18 @@ jobs:
     uses: ./.github/workflows/ci-ansible.yml
     with:
       role: bfd-pipeline
+      bfd_version: ${{ needs.mvn-verify.outputs.BFD_PARENT_VERSION }}
 
   ansible-role-bfd-server:
     needs: mvn-verify
     uses: ./.github/workflows/ci-ansible.yml
     with:
       role: bfd-server
+      bfd_version: ${{ needs.mvn-verify.outputs.BFD_PARENT_VERSION }}
 
   ansible-role-bfd-db-migrator:
     needs: mvn-verify
     uses: ./.github/workflows/ci-ansible.yml
     with:
       role: bfd-db-migrator
+      bfd_version: ${{ needs.mvn-verify.outputs.BFD_PARENT_VERSION }}

apps/.dockerignore

@@ -1,6 +1,6 @@
 *
 !bfd-model/bfd-model-rif/src/main/resources/db/migration
-!bfd-server/bfd-server-launcher/target/bfd-server-launcher-1.0.0-SNAPSHOT.zip
-!bfd-server/bfd-server-war/target/bfd-server-war-1.0.0-SNAPSHOT.war
-!bfd-pipeline/bfd-pipeline-app/target/bfd-pipeline-app-1.0.0-SNAPSHOT.zip
-!bfd-db-migrator/target/bfd-db-migrator-1.0.0-SNAPSHOT.zip
+!bfd-server/bfd-server-launcher/target/bfd-server-launcher-*.zip
+!bfd-server/bfd-server-war/target/bfd-server-war-*.war
+!bfd-pipeline/bfd-pipeline-app/target/bfd-pipeline-app-*.zip
+!bfd-db-migrator/target/bfd-db-migrator-*.zip

apps/build.groovy

@@ -94,10 +94,10 @@ EOF
 	}
 
 	return new AppBuildResults(
-		dbMigratorZip: 'apps/bfd-db-migrator/target/bfd-db-migrator-1.0.0-SNAPSHOT.zip',
-		dataPipelineZip: 'apps/bfd-pipeline/bfd-pipeline-app/target/bfd-pipeline-app-1.0.0-SNAPSHOT.zip',
-		dataServerLauncher: 'apps/bfd-server/bfd-server-launcher/target/bfd-server-launcher-1.0.0-SNAPSHOT.zip',
-		dataServerWar: 'apps/bfd-server/bfd-server-war/target/bfd-server-war-1.0.0-SNAPSHOT.war'
+		dbMigratorZip:      sh(returnStdout: true, script: """find "${workspace}/apps/bfd-db-migrator/target" -type f -name bfd-db-migrator-*.zip""").trim(),
+		dataPipelineZip:    sh(returnStdout: true, script: """find "${workspace}/apps/bfd-pipeline/bfd-pipeline-app/target" -type f -name bfd-pipeline-app-*.zip""").trim(),
+		dataServerLauncher: sh(returnStdout: true, script: """find "${workspace}/apps/bfd-server/bfd-server-launcher/target" -type f -name bfd-server-launcher-*.zip""").trim(),
+		dataServerWar:      sh(returnStdout: true, script: """find "${workspace}/apps/bfd-server/bfd-server-war/target" -type f -name bfd-server-war-*.war""").trim()
 	)
 }
 

ops/ansible/roles/bfd-db-migrator/README.md

@@ -49,7 +49,8 @@ Here's an example of how to apply this role to the `bfd-db-migrator` host in an
         import_role:
             name: bfd-db-migrator
         vars:
-            db_migrator_zip: "{{ lookup('env','HOME') }}/.m2/repository/gov/cms/bfd/bfd-db-migrator/1.0.0-SNAPSHOT/bfd-db-migrator-1.0.0-SNAPSHOT.zip"
+            bfd_version: 2.0.0-SNAPSHOT
+            db_migrator_zip: "{{ lookup('env','HOME') }}/.m2/repository/gov/cms/bfd/bfd-db-migrator/{{ bfd_version }}/bfd-db-migrator-{{ bfd_version }}.zip"
             db_migrator_db_url: 'jdbc:hsqldb:mem:test'
             db_migrator_db_username: "{{ vault_db_migrator_db_username }}"
             db_migrator_db_password: "{{ vault_db_migrator_db_password }}"

ops/ansible/roles/bfd-db-migrator/defaults/main.yml

@@ -1,4 +1,5 @@
 ---
+bfd_version: 1.0.0-SNAPSHOT
 env: test
 db_migrator_dir: /opt/bfd-db-migrator
 db_migrator_user: bb-migrator

ops/ansible/roles/bfd-db-migrator/templates/bfd-db-migrator-service.sh.j2

@@ -31,7 +31,7 @@ export NEW_RELIC_METRIC_PERIOD='{{ db_migrator_new_relic_metric_period }}'
 
 LOGS_DIR='{{ db_migrator_dir }}/'
 
-exec "{{ db_migrator_dir }}/bfd-db-migrator-1.0.0-SNAPSHOT/bfd-db-migrator.sh" \
+exec "{{ db_migrator_dir }}/bfd-db-migrator-{{ bfd_version }}/bfd-db-migrator.sh" \
 	"-DbfdDbMigrator.logs.dir=${LOGS_DIR}" \
 	-Djava.io.tmpdir={{ db_migrator_tmp_dir }} \
 	&>> "{{ db_migrator_dir }}/migrator-log.json"

ops/ansible/roles/bfd-db-migrator/test/run-tests.sh

@@ -5,25 +5,25 @@ set -eou pipefail
 
 function help() {
     echo "This script runs our test cases locally, via Docker."
-    echo "Usage: ${0} [-e extra-vars] [-hk] [image id]"
+    echo "Usage: ${0} [-e extra-vars] [-hk] [-i image] [bfd_version]"
     echo "Options:"
     echo "  ${0} -e <extra-vars>: [e]xtra variables for ansible-playbook."
     echo "  ${0} -h:              [h]elp displays this message and exits."
-    # TODO: complete the getopts implementation. See BFD-1628.
-    # echo "  ${0} -i <ID>: image [i]d set in 'ghcr.io/cmsgov/bfd-apps:<ID>'. Defaults to current commit hash."
-    echo "  ${0} -k:              [k]eeps the container under test instead of removing it. Defaults to removing the container."
+    echo "  ${0} -i <ID>:         [i]mage id set in 'ghcr.io/cmsgov/bfd-apps:<ID>'. Defaults to current commit hash."
+    echo "  ${0} -k:              [k]eep the container under test instead of removing it. Defaults to removing the container."
 }
 
-REMOVE_CONTAINER=true # exported after getopts below...
+# defaults, exported after getopts below...
+REMOVE_CONTAINER=true
+BFD_APPS_IMAGE_ID="$(git rev-parse --short HEAD)"
+
 export ROLE=bfd-db-migrator
 export CONTAINER_NAME="$ROLE"
 export TEST_PLAY=test_basic.yml
 export ANSIBLE_SPEC="ansible"
-export ARTIFACT_DIRECTORY=".m2/repository/gov/cms/bfd/bfd-db-migrator/1.0.0-SNAPSHOT"
-export ARTIFACT="bfd-db-migrator-1.0.0-SNAPSHOT.zip"
 
 # iterate getopts
-while getopts "e:hk" option; do
+while getopts "e:i:hk" option; do
     case "$option" in
       e) # extra-vars
         EXTRA_VARS="$OPTARG"
@@ -32,9 +32,8 @@ while getopts "e:hk" option; do
         help
         exit
         ;;
-      # TODO: complete the getopts implementation. See BFD-1628.
-      # i) # image id
-      #    input_bfd_apps_image_id="$OPTARG";;
+      i) # image id
+        BFD_APPS_IMAGE_ID="$OPTARG";;
       k) # keep container
         REMOVE_CONTAINER=false
         ;;
@@ -46,12 +45,11 @@ while getopts "e:hk" option; do
 done
 shift "$((OPTIND-1))"
 
-# TODO: complete the getopts implementation. See BFD-1628.
-# use the input from option '-i' or default to current commit's short sha
-# export BFD_APPS_IMAGE_ID="${input_bfd_apps_image_id:-$(git rev-parse --short HEAD)}"
-# use input "$1" or default to current commit's short sha
-export BFD_APPS_IMAGE_ID="${1:-$(git rev-parse --short HEAD)}"
-export REMOVE_CONTAINER EXTRA_VARS
+export BFD_VERSION="${1:-1.0.0-SNAPSHOT}"
+export ARTIFACT_DIRECTORY=".m2/repository/gov/cms/bfd/bfd-db-migrator/${BFD_VERSION}"
+export ARTIFACT="bfd-db-migrator-${BFD_VERSION}.zip"
+
+export REMOVE_CONTAINER EXTRA_VARS BFD_APPS_IMAGE_ID
 
 # Determine the directory that this script is in.
 SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"

ops/ansible/roles/bfd-db-migrator/test/test.sh

@@ -19,4 +19,4 @@ source venv/bin/activate
 ansible-playbook "$TEST_PLAY" --inventory=inventory.docker.yaml --syntax-check
 
 # Run the Ansible test case.
-ansible-playbook "$TEST_PLAY" --inventory=inventory.docker.yaml --extra-vars "$EXTRA_VARS"
+ansible-playbook "$TEST_PLAY" --inventory=inventory.docker.yaml --extra-vars "$EXTRA_VARS" --extra-vars bfd_version="$BFD_VERSION"

ops/ansible/roles/bfd-db-migrator/test/test_basic.yml

@@ -21,7 +21,7 @@
         name: bfd-db-migrator
       vars:
         env: dev
-        db_migrator_zip: "{{ lookup('env','HOME') }}/.m2/repository/gov/cms/bfd/bfd-db-migrator/1.0.0-SNAPSHOT/bfd-db-migrator-1.0.0-SNAPSHOT.zip"
+        db_migrator_zip: "{{ lookup('env','HOME') }}/.m2/repository/gov/cms/bfd/bfd-db-migrator/{{ bfd_version }}/bfd-db-migrator-{{ bfd_version }}.zip"
         db_migrator_db_url: jdbc:postgresql://db:5432/fhirdb
         db_migrator_db_username: bfd
         db_migrator_db_password: bfd

ops/ansible/roles/bfd-pipeline/README.md

@@ -20,7 +20,7 @@ Role Variables
 
 This role is highly configurable, though it tries to provide reasonable defaults where possible. Here are the variables that must be defined by users:
 
-    data_pipeline_zip: /home/karlmdavis/workspaces/cms/beneficiary-fhir-data.git/apps/bfd-pipeline/bfd-pipeline-app/target/bfd-pipeline-app-1.0.0-SNAPSHOT.zip
+    data_pipeline_zip: /home/karlmdavis/workspaces/cms/beneficiary-fhir-data.git/apps/bfd-pipeline/bfd-pipeline-app/target/bfd-pipeline-app-2.0.0-SNAPSHOT.zip
     data_pipeline_s3_bucket: name-of-the-s3-bucket-with-the-data-to-process
     data_pipeline_hicn_hash_iterations: 42  # NIST recommends at least 1000
     data_pipeline_hicn_hash_pepper: '6E6F747468657265616C706570706572'  # Hex-encoded "nottherealpepper".
@@ -51,7 +51,7 @@ Here's an example of how to apply this role to the `etlbox` host in an Ansible p
         - include_role:
             name: bfd-pipeline
           vars:
-            data_pipeline_zip: /home/karlmdavis/workspaces/cms/beneficiary-fhir-data.git/apps/bfd-pipeline/bfd-pipeline-app/target/bfd-pipeline-app-1.0.0-SNAPSHOT.zip
+            data_pipeline_zip: /home/karlmdavis/workspaces/cms/beneficiary-fhir-data.git/apps/bfd-pipeline/bfd-pipeline-app/target/bfd-pipeline-app-2.0.0-SNAPSHOT.zip
             data_pipeline_s3_bucket: name-of-the-s3-bucket-with-the-data-to-process
             data_pipeline_hicn_hash_iterations: "{{ vault_data_pipeline_hicn_hash_iterations }}"
             data_pipeline_hicn_hash_pepper: "{{ vault_data_pipeline_hicn_hash_pepper }}"

ops/ansible/roles/bfd-pipeline/defaults/main.yml

@@ -1,4 +1,5 @@
 ---
+bfd_version: 1.0.0-SNAPSHOT
 env_name_std: 'unknown-environment'
 
 data_pipeline_dir: /usr/local/bluebutton-data-pipeline

ops/ansible/roles/bfd-pipeline/templates/bfd-pipeline-service.sh.j2

@@ -77,7 +77,7 @@ export BFD_ENV_NAME='{{ env_name_std }}'
 # Set some additional variables.
 JVM_ARGS='{{ data_pipeline_jvm_args }}'
 
-exec "{{ data_pipeline_dir }}/bfd-pipeline-app-1.0.0-SNAPSHOT/bfd-pipeline-app.sh" \
+exec "{{ data_pipeline_dir }}/bfd-pipeline-app-{{ bfd_version }}/bfd-pipeline-app.sh" \
 	${JVM_ARGS} \
 	-Djava.io.tmpdir={{ data_pipeline_tmp_dir }} -Dorg.jboss.logging.provider=slf4j \
 	&>> "{{ data_pipeline_dir }}/bluebutton-data-pipeline.log"

ops/ansible/roles/bfd-pipeline/test/run-tests.sh

@@ -1,21 +1,55 @@
 #!/usr/bin/env bash
 
-##
-# This script runs our test cases locally, via Docker.
-##
-
 # Stop immediately if any command returns a non-zero result.
 set -eou pipefail
 
-# These variables can be adjusted to change which test is run.
+function help() {
+    echo "This script runs our test cases locally, via Docker."
+    echo "Usage: ${0} [-e extra-vars] [-hk] [-i image] [bfd_version]"
+    echo "Options:"
+    echo "  ${0} -e <extra-vars>: [e]xtra variables for ansible-playbook."
+    echo "  ${0} -h:              [h]elp displays this message and exits."
+    echo "  ${0} -i <ID>:         [i]mage id set in 'ghcr.io/cmsgov/bfd-apps:<ID>'. Defaults to current commit hash."
+    echo "  ${0} -k:              [k]eep the container under test instead of removing it. Defaults to removing the container."
+}
+
+# exported after getopts below...
+REMOVE_CONTAINER=true
+BFD_APPS_IMAGE_ID="$(git rev-parse --short HEAD)"
+
 export ROLE=bfd-pipeline
 export CONTAINER_NAME="$ROLE"
 export TEST_PLAY=test_basic.yml
 export ANSIBLE_SPEC="ansible"
-# use input "$1" or default to current commit's short sha
-export BFD_APPS_IMAGE_ID="${1:-$(git rev-parse --short HEAD)}"
-export ARTIFACT_DIRECTORY=".m2/repository/gov/cms/bfd/bfd-pipeline-app/1.0.0-SNAPSHOT"
-export ARTIFACT="bfd-pipeline-app-1.0.0-SNAPSHOT.zip"
+
+# iterate getopts
+while getopts "e:i:hk" option; do
+    case "$option" in
+      e) # extra-vars
+        EXTRA_VARS="$OPTARG"
+        ;;
+      h) # help
+        help
+        exit
+        ;;
+      i) # image id
+        BFD_APPS_IMAGE_ID="$OPTARG";;
+      k) # keep container
+        REMOVE_CONTAINER=false
+        ;;
+     \?) # Invalid
+        help
+        exit 1
+        ;;
+    esac
+done
+shift "$((OPTIND-1))"
+
+export BFD_VERSION="${1:-1.0.0-SNAPSHOT}"
+export ARTIFACT_DIRECTORY=".m2/repository/gov/cms/bfd/bfd-pipeline-app/${BFD_VERSION}"
+export ARTIFACT="bfd-pipeline-app-${BFD_VERSION}.zip"
+
+export REMOVE_CONTAINER EXTRA_VARS BFD_APPS_IMAGE_ID
 
 # Determine the directory that this script is in.
 SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"

ops/ansible/roles/bfd-pipeline/test/test.sh

@@ -9,7 +9,6 @@ SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 # Run everything from that directory.
 cd "$SCRIPT_DIR"
 
-# Re-activate existing virtualenv or exit 1
 if [ ! -d venv ]; then
   echo 'Error: Missing directory venv.'
   exit 1
@@ -20,7 +19,7 @@ source venv/bin/activate
 ansible-playbook "$TEST_PLAY" --inventory=inventory.docker.yaml --syntax-check
 
 # Run the Ansible test case.
-ansible-playbook "$TEST_PLAY" --inventory=inventory.docker.yaml
+ansible-playbook "$TEST_PLAY" --inventory=inventory.docker.yaml --extra-vars "$EXTRA_VARS" --extra-vars bfd_version="$BFD_VERSION"
 
 # Run the role/playbook again, checking to make sure it's idempotent.
 if ansible-playbook "$TEST_PLAY" --inventory=inventory.docker.yaml | tee /dev/stderr | grep -q "${CONTAINER_NAME}.*changed=0.*failed=0"; then

ops/ansible/roles/bfd-pipeline/test/test_basic.yml

@@ -9,7 +9,7 @@
       import_role:
         name: bfd-pipeline
       vars:
-        data_pipeline_zip: "{{ lookup('env','HOME') }}/.m2/repository/gov/cms/bfd/bfd-pipeline-app/1.0.0-SNAPSHOT/bfd-pipeline-app-1.0.0-SNAPSHOT.zip"
+        data_pipeline_zip: "{{ lookup('env','HOME') }}/.m2/repository/gov/cms/bfd/bfd-pipeline-app/{{ bfd_version }}/bfd-pipeline-app-{{ bfd_version }}.zip"
         data_pipeline_s3_bucket: 'example-fake'  # Doesn't need to actually exist.
         data_pipeline_hicn_hash_iterations: '42'  # NIST recommends at least 1000
         data_pipeline_hicn_hash_pepper: '6E6F747468657265616C706570706572'  # Hex-encoded "nottherealpepper".

ops/ansible/roles/bfd-server/defaults/main.yml

@@ -1,6 +1,6 @@
 ---
-
-# The directory that the Blue Button Data Server will be installed to. 
+bfd_version: 1.0.0-SNAPSHOT
+# The directory that the Blue Button Data Server will be installed to.
 data_server_dir: /usr/local/bfd-server
 
 # The directory that will be set as the 'java.io.tmpdir'

ops/ansible/roles/bfd-server/tasks/install.yml

@@ -79,7 +79,7 @@
 
 - name: Find New Relic Agent
   find:
-    paths: "{{ data_server_dir }}/bfd-server-launcher-1.0.0-SNAPSHOT/lib/"
+    paths: "{{ data_server_dir }}/bfd-server-launcher-{{ bfd_version }}/lib/"
     pattern: newrelic-agent*.jar
   register: find_nra
 

ops/ansible/roles/bfd-server/templates/bfd-server.sh.j2

@@ -4,7 +4,7 @@
 export BFD_PORT='{{ data_server_appserver_https_port }}'
 export BFD_KEYSTORE='{{ data_server_dir }}/bluebutton-appserver-keystore.pfx'
 export BFD_TRUSTSTORE='{{ data_server_dir }}/bluebutton-appserver-truststore.pfx'
-export BFD_WAR='{{ data_server_dir }}/{{ data_server_war | basename }}'
+export BFD_WAR="$(find {{ data_server_dir }} -name 'bfd-server-war-*.war')"
 
 # The WAR picks up its config from Java system properties, so set some variables we can use for
 # those.
@@ -184,7 +184,7 @@ service_startup_check() {
 JVM_ARGS='{{ data_server_appserver_jvmargs }}'
 
 # 3... 2... 1... launch!
-bfd-server-launcher-1.0.0-SNAPSHOT/bfd-server-launcher.sh \
+bfd-server-launcher-{{ bfd_version }}/bfd-server-launcher.sh \
   -javaagent:{{ data_server_dir }}/newrelic/newrelic.jar \
   ${JVM_ARGS} \
   "-DbfdServer.logs.dir=${LOGS_DIR}" \