Merge pull request usdot-jpo-ode#37 from usdot-jpo-ode/candidate_r1

Merge candidate_r1 into master
CDOT-CV · Feb 26, 2024 · 921f0be · 921f0be
2 parents 8dfca7f + a883516
commit 921f0be
Show file tree

Hide file tree

Showing 13 changed files with 279 additions and 94 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -1,33 +1,31 @@
 name: CI
 on: 
   pull_request:
+    types: [opened, reopened, synchronize]
   push:
+    branches: [develop, master]
 
 jobs:
   jpo-sdw-depositor:
    runs-on: ubuntu-latest
    container:
-     image: openjdk:17-jdk-slim-buster
+     image: eclipse-temurin:21-jdk-alpine
      options: --user root
    steps:
      - name: Checkout ${{ github.event.repository.name }}
-       uses: actions/checkout@v3    
+       uses: actions/checkout@v4
      - name: Set up Maven
        uses: stCarolas/[email protected]
        with:
-        maven-version: 3.8.2          
-     - uses: actions/setup-java@v3
-       with:
-         java-version: '11'
-         distribution: 'temurin'               
+        maven-version: 3.9.6
      - name: Build
        run: |
          cd $GITHUB_WORKSPACE
          ls -ls && pwd
          mvn -e -X clean org.jacoco:jacoco-maven-plugin:prepare-agent package
          mvn -e -X clean org.jacoco:jacoco-maven-plugin:report package
      - name: Archive code coverage results
-       uses: actions/upload-artifact@v3
+       uses: actions/upload-artifact@v4
        with:
          name: jpo-sdw-depositor
          path: /__w/jpo-sdw-depositor/jpo-sdw-depositor/target 
@@ -37,22 +35,20 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout ${{ github.event.repository.name }}
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           fetch-depth: 0   
       - name: Download code coverage results
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
         with:
           name: jpo-sdw-depositor
           path: home/runner/work/jpo-sdw-depositor/jpo-sdw-depositor/target     
       - name: Find jacoco.xml
         shell: bash
         run: |
-          find "$GITHUB_WORKSPACE" -name "jacoco.xml"                
+          find "$GITHUB_WORKSPACE" -name "jacoco.xml"  
       - name: Setup SonarScanner
-        uses: warchant/setup-sonar-scanner@v4
-        with:
-          version: 4.8.0.2856
+        uses: warchant/setup-sonar-scanner@v7
       - name: Generate sonar properties file
         run: |
           cat <<EOF > /tmp/sonar-scanner.properties
@@ -72,4 +68,4 @@ jobs:
         with:
           sonar-properties-path: /tmp/sonar-scanner.properties
           sonar-token: ${{ secrets.SONAR_TOKEN }}
-          working-dir: $GITHUB_WORKSPACE                 
+          working-dir: $GITHUB_WORKSPACE                 
diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml
@@ -1,11 +1,6 @@
 name: Docker build
 
 on:
-  push:
-    branches-ignore:
-      - "develop"
-      - "master"
-      - "release/*"
   pull_request:
       types: [opened, synchronize, reopened]
 
@@ -14,9 +9,8 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
-      - run: ls -la & pwd
+        uses: docker/setup-buildx-action@v3
       - name: Build
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v5
diff --git a/.github/workflows/dockerhub.yml b/.github/workflows/dockerhub.yml
@@ -12,16 +12,16 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
+        uses: docker/setup-buildx-action@v3
       - name: Login to DockerHub
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
       - name: Build
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v5
         with:
           push: true
           tags: usdotjpoode/jpo-sdw-depositor:${{ github.ref_name }}
diff --git a/Dockerfile b/Dockerfile
@@ -1,5 +1,5 @@
 # Build container
-FROM maven:3.5.4-jdk-8-alpine as builder
+FROM maven:3.8-eclipse-temurin-21-alpine as builder
 MAINTAINER [email protected]
 
 WORKDIR /home
@@ -11,11 +11,11 @@ COPY ./src ./src
 RUN mvn clean package -DskipTests
 
 # Run container
-FROM eclipse-temurin:11-jre-alpine
+FROM eclipse-temurin:21-jre-alpine
 
 WORKDIR /home
-COPY --from=builder /home/target/jpo-sdw-depositor-1.3.0.jar /home
+COPY --from=builder /home/target/jpo-sdw-depositor-1.6.0-SNAPSHOT.jar /home
 
 ENTRYPOINT ["java", \
 	"-jar", \
-	"/home/jpo-sdw-depositor-1.3.0.jar"]
+	"/home/jpo-sdw-depositor-1.6.0-SNAPSHOT.jar"]
diff --git a/README.md b/README.md
@@ -86,3 +86,10 @@ mvn test
 ```
 
 It should be noted that Maven & Java are required to run the unit tests. If you do not have Maven or Java installed, you can reopen the project in the provided dev container and run the tests from there.
+
+# Object data consumption
+The KafkaConsumerRestDepositor will accept any string as input to be passed into the SDW. If provided a JSON object, the tokens of "encodedMsg" and "estimatedRemovalDate" will be passed through directly to the SDW in the form of the following:
+{depositRequests:[{"encodeType": STRING ,"encodedMsg": STRING, "estimatedRemovalDate": STRING}]}
+
+If provided a string of non-json form, the value of "encodedMsg" will inherit the passed value and information will be passed to the SDW in the form of the following:
+{depositRequests:[{"encodeType": STRING ,"encodedMsg": STRING}]}
diff --git a/docs/Release_notes.md b/docs/Release_notes.md
@@ -1,6 +1,20 @@
 Jpo-sdw-depositor Release Notes
 ----------------------------
 
+Version 1.6.0, released February 2024
+----------------------------------------
+
+### **Summary**
+The changes for the jpo-sdw-depositor 1.6.0 release include a java update & dockerhub image documentation.
+
+Enhancements in this release:
+- CDOT PR 12: Updated Java to v21
+- CDOT PR 11: Added dockerhub image documentation
+
+Known Issues:
+- No known issues at this time.
+
+
 Version 1.5.0, released November 2023
 ----------------------------------------
 

diff --git a/docs/dockerhub.md b/docs/dockerhub.md
@@ -0,0 +1,79 @@
+# jpo-sdw-depositor
+
+## GitHub Repository Link
+https://github.com/usdot-jpo-ode/jpo-sdw-depositor
+
+## Purpose
+The purpose of the jpo-sdw-depositor program is to deposit messages to the SDX.
+
+## How to pull the latest image
+The latest image can be pulled using the following command:
+> docker pull usdotjpoode/jpo-sdw-depositor:develop
+
+## Required environment variables
+- DOCKER_HOST_IP
+- SDW_EMAIL_LIST
+- SDW_EMAIL_FROM
+- SDW_API_KEY
+
+## Direct Dependencies
+The SDWD will fail to start up if the following containers are not already present:
+- Kafka
+- Zookeeper (relied on by Kafka)
+
+## Indirect Dependencies
+The SDWD will not receive messages to process if the ODE is not running.
+
+## Example docker-compose.yml with direct dependencies:
+```
+version: '2'
+services:
+  zookeeper:
+    image: wurstmeister/zookeeper
+    ports:
+      - "2181:2181"
+
+  kafka:
+    image: wurstmeister/kafka
+    ports:
+      - "9092:9092"
+    environment:
+      KAFKA_ADVERTISED_HOST_NAME: ${DOCKER_HOST_IP}
+      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
+      KAFKA_CREATE_TOPICS: "test:1:1"
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock
+      
+  sdw_depositor:
+    image: usdotjpoode/jpo-sdw-depositor:release_q3
+    environment:
+      # required
+      DOCKER_HOST_IP: ${DOCKER_HOST_IP}
+      SDW_EMAIL_LIST: ${SDW_EMAIL_LIST}
+      SDW_EMAIL_FROM: ${SDW_EMAIL_FROM}
+      SDW_API_KEY: ${SDW_API_KEY}
+      # optional
+      SDW_DESTINATION_URL: ${SDW_DESTINATION_URL}
+      SPRING_MAIL_HOST: ${SPRING_MAIL_HOST}
+      SPRING_MAIL_PORT: ${SPRING_MAIL_PORT}
+      SDW_SUBSCRIPTION_TOPIC: ${SDW_SUBSCRIPTION_TOPIC}
+    logging:
+      options:
+        max-size: "10m"
+        max-file: "5"
+```
+
+## Expected startup output
+The latest logs should look something like this:
+```
+2023-11-09 17:40:31.082  INFO 1 --- [           main] o.a.k.c.c.internals.AbstractCoordinator  : [Consumer clientId=consumer-usdot.jpo.sdw-1, groupId=usdot.jpo.sdw] Discovered group coordinator 192.168.0.243:9092 (id: 2147482646 rack: null)
+2023-11-09 17:40:31.089  INFO 1 --- [           main] o.a.k.c.c.internals.AbstractCoordinator  : [Consumer clientId=consumer-usdot.jpo.sdw-1, groupId=usdot.jpo.sdw] (Re-)joining group
+2023-11-09 17:40:31.174  INFO 1 --- [           main] o.a.k.c.c.internals.AbstractCoordinator  : [Consumer clientId=consumer-usdot.jpo.sdw-1, groupId=usdot.jpo.sdw] (Re-)joining group
+2023-11-09 17:40:31.227  INFO 1 --- [           main] o.a.k.c.c.internals.AbstractCoordinator  : [Consumer clientId=consumer-usdot.jpo.sdw-1, groupId=usdot.jpo.sdw] Successfully joined group with generation Generation{generationId=1, memberId='consumer-usdot.jpo.sdw-1-cd6afbec-5bfe-46e0-beb6-539d47426902', protocol='range'}
+2023-11-09 17:40:31.233  INFO 1 --- [           main] o.a.k.c.c.internals.ConsumerCoordinator  : [Consumer clientId=consumer-usdot.jpo.sdw-1, groupId=usdot.jpo.sdw] Finished assignment for group at generation 1: {consumer-usdot.jpo.sdw-1-cd6afbec-5bfe-46e0-beb6-539d47426902=Assignment(partitions=[topic.SDWDepositorInput-0])}
+2023-11-09 17:40:31.369  INFO 1 --- [           main] o.a.k.c.c.internals.AbstractCoordinator  : [Consumer clientId=consumer-usdot.jpo.sdw-1, groupId=usdot.jpo.sdw] Successfully synced group in generation Generation{generationId=1, memberId='consumer-usdot.jpo.sdw-1-cd6afbec-5bfe-46e0-beb6-539d47426902', protocol='range'}
+2023-11-09 17:40:31.370  INFO 1 --- [           main] o.a.k.c.c.internals.ConsumerCoordinator  : [Consumer clientId=consumer-usdot.jpo.sdw-1, groupId=usdot.jpo.sdw] Notifying assignor about the new Assignment(partitions=[topic.SDWDepositorInput-0])
+2023-11-09 17:40:31.378  INFO 1 --- [           main] o.a.k.c.c.internals.ConsumerCoordinator  : [Consumer clientId=consumer-usdot.jpo.sdw-1, groupId=usdot.jpo.sdw] Adding newly assigned partitions: topic.SDWDepositorInput-0
+2023-11-09 17:40:31.415  INFO 1 --- [           main] o.a.k.c.c.internals.ConsumerCoordinator  : [Consumer clientId=consumer-usdot.jpo.sdw-1, groupId=usdot.jpo.sdw] Found no committed offset for partition topic.SDWDepositorInput-0
+2023-11-09 17:40:31.454  INFO 1 --- [           main] o.a.k.c.c.internals.SubscriptionState    : [Consumer clientId=consumer-usdot.jpo.sdw-1, groupId=usdot.jpo.sdw] Resetting offset for partition topic.SDWDepositorInput-0 to position FetchPosition{offset=0, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[192.168.0.243:9092 (id: 1001 rack: null)], epoch=0}}.
+```