Update orb

README.md

@@ -1,10 +1,30 @@
 # Sumologic Orb
-https://circleci.com/developer/orbs/orb/sumologic/sumologic?version=2.2.1
-
 Easily capture analytics from your CircleCI jobs in your Sumologic dashboard!
 
+
+## Setup
+
+To configure the orb follow the instructions in [Circle CI for build and deploy](https://help.sumologic.com/docs/observability/sdo/set-up-sdo/sdo-manual-configuration/#circleci-for-build-and-deploy) docs.
+
+To know more about the parameters and example configuration refer [Sumologic Orb documentation](https://circleci.com/developer/en/orbs/orb/sumologic/sumologic).
+
 ## job-collector
 Add this command to your job with environment, team or service custom-data values as parameters. This command will run with the rest of the commands of the job for sending job log. This command has been introduced so as to send environment, team and service information at job level. If this command is not included at job level in one of it's steps, then the workflow-collector job will send the job log. Using it without the parameters will result in empty custom-data values being sent to Sumo.
 
 ## workflow-collector
-Add this job to your workflow with no require statements. This job will run in parallel with the rest of your workflow for monitoring and will exit when all other jobs have completed. Custom data can be supplied via the custom-data parameter in the form of valid JSON. Keys and values can be supplied literally or as environment variables, though supplying values as additional, nested JSON is not supported.
+Add this job to your workflow with no require statements. This job will run in parallel with the rest of your workflow for monitoring the health of all jobs in the Workflow. It exits when all other jobs have completed. Custom data can be supplied via the custom-data parameter in the form of valid JSON. Keys and values can be supplied literally or as environment variables, though supplying values as additional, nested JSON is not supported. Use `timeout-seconds` parameter (defaults to 180s/3mins) to end the workflow-collector early when required.
+
+## Testing
+
+* [Install](https://circleci.com/docs/local-cli/#installation) and [configure](https://circleci.com/docs/local-cli/#configure-the-cli) the CircleCI CLI Tool.
+* brew install yamllint shellcheck
+* Run the [validation](https://circleci.com/docs/testing-orbs/#validation) commands
+
+    `yamllint ./src`
+    `circleci orb validate orb.yml`
+    `shellcheck src/scripts/*.sh`
+
+
+
+
+## Releasing

src/examples/workflow-collector.yml

@@ -3,7 +3,7 @@ description: |
 usage:
     version: "2.1"
     orbs:
-        sumologic: sumologic/[email protected].1
+        sumologic: sumologic/[email protected].2
     jobs:
         build:
             docker:

src/scripts/job-collector.sh

@@ -1,24 +1,53 @@
 #!/bin/bash
 VCS_SHORT=$(echo "$CIRCLE_BUILD_URL" | cut -d"/" -f4)
 case "$VCS_SHORT" in
-gh)
+    gh)
+    # For GitHub OAuth App integration type
     VCS=github
     ;;
-bb)
+    bb)
+    # For GitHub OAuth App integration type
     VCS=bitbucket
     ;;
-*)
+    circleci)
+    # For GitHub App and Gitlab  integration type
+    VCS=circleci
+    ;;
+    *)
     echo "No VCS found. Error" && exit 1
     ;;
 esac
 JOB_DATA_RAW=$(curl -s "https://circleci.com/api/v1.1/project/$VCS/$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME/$CIRCLE_BUILD_NUM?circle-token=${CIRCLE_TOKEN}")
+
+if [[ "$JOB_DATA_RAW" =~ "Invalid token" ]];
+then
+    echo "Your circle-token parameter may be wrong Error: $JOB_DATA_RAW"
+    exit 1
+fi
+
+if [[ -z "${JOB_DATA_RAW#"${JOB_DATA_RAW%%[! ]*}"}" ]];
+then
+    echo "No Job Found for $VCS/$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME/$CIRCLE_BUILD_NUM"
+    exit 1
+fi
+
 # removing steps and circle_yml keys from object
 JOB_DATA_RAW=$(echo "$JOB_DATA_RAW" | jq 'del(.circle_yml)' | jq 'del(.steps)')
 JOB_NAME=$(echo "$JOB_DATA_RAW" | jq .workflows | jq .job_name)
+CMD_STATUS=$?
+if [ "$CMD_STATUS" -ne 0 ] ; then
+   echo "Error in parsing payload: $VCS/$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME/$CIRCLE_BUILD_NUM error: $JOB_DATA_RAW"
+   exit 1
+fi
+
 JOB_STATUS=$(echo "$JOB_DATA_RAW" | jq .status)
+
 echo "JOB: $JOB_NAME"
 echo "JOB NUM: $CIRCLE_BUILD_NUM"
 echo "STATUS: $JOB_STATUS"
+
+
+
 #####
 # Send Job Data to SumoLogic
 #####
@@ -39,5 +68,5 @@ else
     echo "No valid custom data found to append to the job data"
 fi
 echo "$JOB_DATA_RAW" > /tmp/sumologic-logs/job-collector.json
-curl -s -X POST -T /tmp/sumologic-logs/job-collector.json "${JOB_HTTP_SOURCE}"
-echo "Job details sent to Sumo."
+curl -s -w "SumoHTTPSendStatus: %{http_code}\n" -X POST -T /tmp/sumologic-logs/job-collector.json "${JOB_HTTP_SOURCE}"
+echo "Job details sent to Sumo."

src/scripts/workflow-collector.sh

@@ -1,32 +1,31 @@
 #!/bin/bash
 WF_DATA=$(curl -s "https://circleci.com/api/v2/workflow/$CIRCLE_WORKFLOW_ID/job?circle-token=${CIRCLE_TOKEN}")
-WF_ITEMS=$(echo "$WF_DATA" | jq '.items')
-WF_LENGTH=$(echo "$WF_ITEMS" | jq length)
-WF_MESSAGE=$(echo "$WF_DATA" | jq '.message')
-
 # Exit if no Workflow.
+if [[ "$WF_DATA" =~ "Invalid token" ]];
+then
+    echo "Your circle-token parameter may be wrong Error: $WF_DATA"
+    exit 1
+fi
+WF_MESSAGE=$(echo "$WF_DATA" | jq '.message' 2>&1)
+CMD_STATUS=$?
+if [ "$CMD_STATUS" -ne 0 ] ; then
+   echo "Error in parsing payload of workflow jobs: $CIRCLE_WORKFLOW_ID/job error: $WF_DATA" && exit 1
+fi
 if [ "$WF_MESSAGE" = "\"Workflow not found\"" ];
 then
     echo "No Workflow was found."
     echo "Your circle-token parameter may be wrong or you do not have access to this Workflow."
     exit 1
 fi
-
-VCS_SHORT=$(echo "$CIRCLE_BUILD_URL" | cut -d"/" -f4)
-case "$VCS_SHORT" in
-    gh)
-    VCS=github
-    ;;
-    bb)
-    VCS=bitbucket
-    ;;
-    *)
-    echo "No VCS found. Error" && exit 1
-    ;;
-esac
+WF_ITEMS=$(echo "$WF_DATA" | jq '.items')
 
 # Get the current state of all jobs.
-WF_SL_PAYLOAD=$(curl -s "https://circleci.com/api/v2/workflow/$CIRCLE_WORKFLOW_ID?circle-token=${CIRCLE_TOKEN}" | jq '.')
+WF_SL_PAYLOAD_RAW=$(curl -s "https://circleci.com/api/v2/workflow/$CIRCLE_WORKFLOW_ID?circle-token=${CIRCLE_TOKEN}")
+WF_SL_PAYLOAD=$(echo "$WF_SL_PAYLOAD_RAW" | jq '.' 2>&1)
+CMD_STATUS=$?
+if [ "$CMD_STATUS" -ne 0 ] ; then
+   echo "Error in parsing payload of workflow: $CIRCLE_WORKFLOW_ID error: $WF_SL_PAYLOAD_RAW" && exit 1
+fi
 
 # Append any custom data to the workflow data
 ESCAPED_JSON=$(echo "${PARAM_CUSTOMDATA}" | sed -E 's/([^\]|^)"/\1\\"/g')
@@ -39,35 +38,49 @@ else
     echo "No valid custom data found to append to the workflow data"
 fi
 
-echo "Sending current Workflow state to Sumo"
-curl -i -H "Accept: application/json" -H "Content-Type:application/json" -X POST --data "$WF_SL_PAYLOAD" "${WORKFLOW_HTTP_SOURCE}"
+echo "Sending current Workflow state logs to SumoLogic"
+curl -s -w "SumoHTTPSendStatus: %{http_code}\n" -H "Accept: application/json" -H "Content-Type:application/json" -X POST --data "$WF_SL_PAYLOAD" "${WORKFLOW_HTTP_SOURCE}"
 
 declare -A job_status_array
 # Set FIRST_RUN to true to ensure all initial updates are sent.
 FIRST_RUN=true
 # While we still have jobs which are runnung.
 TIMEOUT=$(date -d "${PARAM_TIMEOUT_SECONDS} seconds")
+counter=0
 while true
 do
   if [[ $(date) > $TIMEOUT ]];
   then
     echo "Monitoring loop exceeded timeout of $PARAM_TIMEOUT_SECONDS seconds. Breaking loop and ending the job."
     break
   fi
-
-  WF_SL_PAYLOAD=$(curl -s "https://circleci.com/api/v2/workflow/$CIRCLE_WORKFLOW_ID?circle-token=${CIRCLE_TOKEN}" | jq '.')
+  counter=$((counter+1))
+  WF_SL_PAYLOAD_RAW=$(curl -s "https://circleci.com/api/v2/workflow/$CIRCLE_WORKFLOW_ID?circle-token=${CIRCLE_TOKEN}")
+  WF_SL_PAYLOAD=$(echo "$WF_SL_PAYLOAD_RAW" | jq '.' 2>&1)
+  CMD_STATUS=$?
+  if [ "$CMD_STATUS" -ne 0 ] ; then
+    echo "Error in parsing payload of workflow: $CIRCLE_WORKFLOW_ID error: $WF_SL_PAYLOAD_RAW counter: $counter"
+    continue
+  fi
   WF_STATUS=$(echo "$WF_SL_PAYLOAD" | jq -r ".status")
 
-  if [[ "$WF_STATUS" != "running" ]];
+  if [[ "$WF_STATUS" != "running" ]] && [[ "$FIRST_RUN" == false ]];
   then
     echo "Workflow status no longer running. Now: ${WF_STATUS}. Breaking loop."
     break
   fi
 
   WF_DATA=$(curl -s "https://circleci.com/api/v2/workflow/$CIRCLE_WORKFLOW_ID/job?circle-token=${CIRCLE_TOKEN}")
-  WF_ITEMS=$(echo "$WF_DATA" | jq '.items')
+  WF_ITEMS=$(echo "$WF_DATA" | jq '.items' 2>&1)
+  CMD_STATUS=$?
+  if [ "$CMD_STATUS" -ne 0 ] ; then
+     echo "Error in parsing payload of workflow jobs: $CIRCLE_WORKFLOW_ID/job error: $WF_DATA counter: $counter"
+     continue
+  fi
   WF_LENGTH=$(echo "$WF_ITEMS" | jq length)
 
+  echo "Found $WF_LENGTH Jobs for workflow: $CIRCLE_WORKFLOW_ID."
+
   # Check all jobs.
   i="0"
   while [ $i -lt "$WF_LENGTH" ]
@@ -76,11 +89,13 @@ do
     JOB_NUMBER=$(echo "$JOB_DATA" | jq -r ".job_number")
     JOB_STATUS=$(echo "$JOB_DATA" | jq -r '.status')
     JOB_NAME=$(echo "$JOB_DATA" | jq -r ".name")
-    if [[ "${JOB_NAME}" != "workflow-collector" ]];
+    PROJECT_SLUG=$(echo "$JOB_DATA" | jq -r ".project_slug")
+
+    if [[ ! "${JOB_NAME}" =~ "workflow-collector" ]];
     then
       if ! [ "${job_status_array["${JOB_NAME}"]}" ];
       then
-        echo "Job '$JOB_NAME' (job number: '$JOB_NUMBER') not tracked, adding to array with status of '$JOB_STATUS'."
+        echo "Job '$JOB_NAME' of project '$PROJECT_SLUG' (job number: '$JOB_NUMBER') not tracked, adding to array with status of '$JOB_STATUS'."
         job_status_array["${JOB_NAME}"]=$JOB_STATUS
       fi
 
@@ -89,8 +104,15 @@ do
         echo "'$JOB_NAME' has a job number of 'null' and status of '$JOB_STATUS'. What's gone wrong?"
       elif [[ "$JOB_NUMBER" != "null" ]];
       then
-        JOB_DATA_RAW=$(curl -s "https://circleci.com/api/v1.1/project/$VCS/$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME/$JOB_NUMBER?circle-token=${CIRCLE_TOKEN}")
-        JOB_STATUS=$(echo "$JOB_DATA_RAW" | jq -r '.status')
+        # Todo migrate to v2 api currently uses older api version https://circleci.com/docs/api/v1/index.html#jobs
+        # Currently v2 api does not contain step details https://discuss.circleci.com/t/circleci-v2-api-job-step/50937
+        JOB_DATA_RAW=$(curl -s "https://circleci.com/api/v1.1/project/$PROJECT_SLUG/$JOB_NUMBER?circle-token=${CIRCLE_TOKEN}")
+        JOB_STATUS=$(echo "$JOB_DATA_RAW" | jq -r '.status' 2>&1)
+        CMD_STATUS=$?
+        if [ "$CMD_STATUS" -ne 0 ] ; then
+           echo "Error in parsing payload of single job: $PROJECT_SLUG/$JOB_NUMBER error: $JOB_DATA_RAW counter: $counter"
+           continue
+        fi
         # Manually set job name as it is currently null
         JOB_DATA_RAW=$(echo "$JOB_DATA_RAW" | jq --arg JOBNAME "$JOB_NAME" '.job_name = $JOBNAME')
         JOB_STEP_NAMES=$(echo "$JOB_DATA_RAW" | jq '.steps' | jq .[] | jq '.name')
@@ -104,19 +126,21 @@ do
           # Handle changes in state.
           if [[ "${job_status_array["${JOB_NAME}"]}" != "$JOB_STATUS" ]] || $FIRST_RUN; then
             # Send update in status to SumoLogic
-            echo "Job '$JOB_NAME' status has changed '${job_status_array["${JOB_NAME}"]}' -> '$JOB_STATUS'. Sending update to SumoLogic."
+            echo "Job '$JOB_NAME' status has changed '${job_status_array["${JOB_NAME}"]}' -> '$JOB_STATUS' FirstTimeRunning: '$FIRST_RUN'. Sending Job state logs to SumoLogic."
             JOB_DATA_RAW=$(echo "$JOB_DATA_RAW" | jq -c '.')
             if [[ -n "${PARAM_CUSTOMDATA}" ]] && echo "$CUSTOM_DATA" | jq -e;
             then
                 JOB_DATA_RAW=$(echo "$JOB_DATA_RAW" | jq -c ". +  {\"custom_data\": $CUSTOM_DATA}")
             else
                 echo "No valid custom data found to append to the job data."
             fi
-            curl -i -H "Accept: application/json" -H "Content-Type:application/json" -X POST --data "$JOB_DATA_RAW" "${JOB_HTTP_SOURCE}"
+            curl -s -w "SumoHTTPSendStatus: %{http_code}\n" -H "Accept: application/json" -H "Content-Type:application/json" -X POST --data "$JOB_DATA_RAW" "${JOB_HTTP_SOURCE}"
           fi
           job_status_array["${JOB_NAME}"]="$JOB_STATUS"
         fi
       fi
+    else
+        echo "Ignoring ${JOB_NAME} job - skipping sending an update to SumoLogic"
     fi
     i="$((i+1))"
     # echo "Incremented loop to $i. Continuing..."
@@ -136,7 +160,7 @@ do
         break
     fi
     if [[ "${job_status_array[$k]}" == "running" ]]; then
-      if [[ "$k" != "workflow-collector" ]]; then
+      if [[ ! "$k" =~ "workflow-collector" ]]; then
         FINISHED=false
         break
       fi
@@ -147,7 +171,13 @@ do
     echo "All jobs are in non running state other than the workflow-collector."
 
     # Get the final state of all jobs.
-    WF_SL_PAYLOAD=$(curl -s "https://circleci.com/api/v2/workflow/$CIRCLE_WORKFLOW_ID?circle-token=${CIRCLE_TOKEN}" | jq '.')
+    WF_SL_PAYLOAD_RAW=$(curl -s "https://circleci.com/api/v2/workflow/$CIRCLE_WORKFLOW_ID?circle-token=${CIRCLE_TOKEN}")
+    WF_SL_PAYLOAD=$(echo "$WF_SL_PAYLOAD_RAW" | jq '.' 2>&1)
+    CMD_STATUS=$?
+    if [ "$CMD_STATUS" -ne 0 ] ; then
+       echo "Error in parsing payload of workflow: $CIRCLE_WORKFLOW_ID error: $WF_SL_PAYLOAD_RAW counter: $counter"
+       continue
+    fi
 
     # Append any custom data to the workflow data
     ESCAPED_JSON=$(echo "${PARAM_CUSTOMDATA}" | sed -E 's/([^\]|^)"/\1\\"/g')
@@ -180,11 +210,11 @@ do
       WF_SL_PAYLOAD=$(echo "$WF_SL_PAYLOAD" | jq -c --arg STOPPED_AT "$STOPPED_AT" '.stopped_at = $STOPPED_AT')
     fi
 
-    echo "Sending final Workflow state to Sumo"
-    curl -i -H "Accept: application/json" -H "Content-Type:application/json" -X POST --data "$WF_SL_PAYLOAD" -s "${WORKFLOW_HTTP_SOURCE}"
+    echo "Sending final Workflow state logs to SumoLogic"
+    curl -s -w "SumoHTTPSendStatus: %{http_code}\n" -H "Accept: application/json" -H "Content-Type:application/json" -X POST --data "$WF_SL_PAYLOAD" -s "${WORKFLOW_HTTP_SOURCE}"
     echo "Finishing up."
     break
   else
     echo "Some jobs are still queued or in progress. Continuing..."
   fi
-done
+done