Snow processing devops prep

.github/workflows/deployment.yml

@@ -188,6 +188,12 @@ jobs:
           oc login "${{ secrets.OPENSHIFT_CLUSTER }}" --token="${{ secrets.OC4_DEV_TOKEN }}"
           PROJ_DEV="e1e498-dev" bash openshift/scripts/oc_provision_noaa_nam_cronjob.sh ${SUFFIX} apply
 
+      - name: VIIRS SNOW cronjob
+        shell: bash
+        run: |
+          oc login "${{ secrets.OPENSHIFT_CLUSTER }}" --token="${{ secrets.OC4_DEV_TOKEN }}"
+          PROJ_DEV="e1e498-dev" bash openshift/scripts/oc_provision_viirs_snow_cronjob.sh ${SUFFIX} apply
+
   # TODO: Delete once pmtiles has run for some time
   # deploy-tileserv:
   #   name: Deploy tileserv to Dev

api/app/jobs/viirs_snow.py

@@ -0,0 +1,12 @@
+import logging
+
+from app import configure_logging
+
+logger = logging.getLogger(__name__)
+
+def main():
+    logger.info("***************Cron job started**************")
+
+if __name__ == '__main__':
+    configure_logging()
+    main()

openshift/scripts/oc_provision_viirs_snow_cronjob.sh

@@ -0,0 +1,57 @@
+#!/bin/sh -l
+#
+source "$(dirname ${0})/common/common"
+
+#%
+#% OpenShift Deploy Helper
+#%
+#%   Intended for use with a pull request-based pipeline.
+#%   Suffixes incl.: pr-###.
+#%
+#% Usage:
+#%
+#%    ${THIS_FILE} [SUFFIX] [apply]
+#%
+#% Examples:
+#%
+#%   Provide a PR number. Defaults to a dry-run.
+#%   ${THIS_FILE} pr-0
+#%
+#%   Apply when satisfied.
+#%   ${THIS_FILE} pr-0 apply
+#%
+
+# Target project override for Dev or Prod deployments
+#
+PROJ_TARGET="${PROJ_TARGET:-${PROJ_DEV}}"
+
+# Specify a default schedule to run daily at 4am
+SCHEDULE="${SCHEDULE:-$((3 + $RANDOM % 54)) 4 * * *}"
+
+# Process template
+OC_PROCESS="oc -n ${PROJ_TARGET} process -f ${TEMPLATE_PATH}/viirs_snow.cronjob.yaml \
+-p JOB_NAME=viirs-snow-${APP_NAME}-${SUFFIX} \
+-p APP_LABEL=${APP_NAME}-${SUFFIX} \
+-p NAME=${APP_NAME} \
+-p SUFFIX=${SUFFIX} \
+-p SCHEDULE=\"${SCHEDULE}\" \
+-p POSTGRES_USER=${POSTGRES_USER:-${APP_NAME}} \
+-p POSTGRES_DATABASE=${POSTGRES_DATABASE:-${APP_NAME}} \
+-p POSTGRES_WRITE_HOST=${POSTGRES_WRITE_HOST:-"patroni-${APP_NAME}-${SUFFIX}-leader"} \
+-p POSTGRES_READ_HOST=${POSTGRES_READ_HOST:-"patroni-${APP_NAME}-${SUFFIX}-replica"} \
+${PROJ_TOOLS:+ "-p PROJ_TOOLS=${PROJ_TOOLS}"} \
+${IMAGE_REGISTRY:+ "-p IMAGE_REGISTRY=${IMAGE_REGISTRY}"}"
+
+# Apply template (apply or use --dry-run)
+#
+OC_APPLY="oc -n ${PROJ_TARGET} apply -f -"
+[ "${APPLY}" ] || OC_APPLY="${OC_APPLY} --dry-run"
+
+# Execute commands
+#
+eval "${OC_PROCESS}"
+eval "${OC_PROCESS} | ${OC_APPLY}"
+
+# Provide oc command instruction
+#
+display_helper "${OC_PROCESS} | ${OC_APPLY}"

openshift/templates/viirs_snow.cronjob.yaml

@@ -0,0 +1,124 @@
+kind: Template
+apiVersion: template.openshift.io/v1
+metadata:
+  name: ${JOB_NAME}-cronjob-template
+  annotations:
+    description: "Scheduled task to download and process VIIRS snow data from the Nasa Snow and Ice Data Centre."
+    tags: "cronjob,viirs,snow"
+labels:
+  app.kubernetes.io/part-of: "${NAME}"
+  app: ${NAME}-${SUFFIX}
+parameters:
+  - name: NAME
+    description: Module name
+    value: wps
+  - name: GLOBAL_NAME
+    description: Name of global Module
+    value: wps-global
+  - name: SUFFIX
+    description: Deployment suffix, e.g. pr-###
+    required: true
+  - name: PROJ_TOOLS
+    value: e1e498-tools
+  - name: JOB_NAME
+    value: viirs-snow
+  - name: IMAGE_REGISTRY
+    required: true
+    value: image-registry.openshift-image-registry.svc:5000
+  - name: POSTGRES_WRITE_HOST
+    required: true
+    value: patroni-${NAME}-${SUFFIX}-leader
+  - name: POSTGRES_READ_HOST
+    value: patroni-${NAME}-${SUFFIX}-replica
+  - name: POSTGRES_USER
+    required: true
+    value: ${NAME}-${SUFFIX}
+  - name: POSTGRES_DATABASE
+    required: true
+    value: ${NAME}-${SUFFIX}
+  - name: SCHEDULE
+    required: true
+  - name: APP_LABEL
+    required: true
+objects:
+  - kind: CronJob
+    apiVersion: batch/v1
+    metadata:
+      name: ${JOB_NAME}
+    spec:
+      schedule: ${SCHEDULE}
+      # We use the "Replace" policy, because we never want the cronjobs to run concurrently,
+      # and if for whatever reason a cronjob gets stuck, we want the next run to proceed.
+      # If we were to use Forbid, and a cronjob gets stuck, then we'd stop gathering data until someone
+      # noticed. We don't want that.
+      concurrencyPolicy: "Replace"
+      jobTemplate:
+        metadata:
+          labels:
+            cronjob: ${JOB_NAME}
+            app: ${APP_LABEL}
+        spec:
+          template:
+            spec:
+              containers:
+                - name: ${JOB_NAME}
+                  image: ${IMAGE_REGISTRY}/${PROJ_TOOLS}/${NAME}-api-${SUFFIX}:${SUFFIX}
+                  imagePullPolicy: "Always"
+                  command:
+                    ["poetry", "run", "python", "-m", "app.jobs.viirs_snow"]
+                  env:
+                    - name: POSTGRES_READ_USER
+                      value: ${POSTGRES_USER}
+                    - name: POSTGRES_WRITE_USER
+                      value: ${POSTGRES_USER}
+                    - name: POSTGRES_PASSWORD
+                      valueFrom:
+                        secretKeyRef:
+                          name: ${GLOBAL_NAME}
+                          key: app-db-password
+                    - name: POSTGRES_WRITE_HOST
+                      value: ${POSTGRES_WRITE_HOST}
+                    - name: POSTGRES_READ_HOST
+                      value: ${POSTGRES_READ_HOST}
+                    - name: POSTGRES_PORT
+                      value: "5432"
+                    - name: POSTGRES_DATABASE
+                      value: ${POSTGRES_DATABASE}
+                    - name: ROCKET_URL_POST_MESSAGE
+                      valueFrom:
+                        configMapKeyRef:
+                          name: ${GLOBAL_NAME}
+                          key: rocket.chat-url-post-message
+                    - name: ROCKET_CHANNEL
+                      valueFrom:
+                        configMapKeyRef:
+                          name: ${GLOBAL_NAME}
+                          key: rocket.chat-channel
+                    - name: ROCKET_USER_ID
+                      valueFrom:
+                        secretKeyRef:
+                          name: ${GLOBAL_NAME}
+                          key: rocket.chat-user-id-secret
+                    - name: ROCKET_AUTH_TOKEN
+                      valueFrom:
+                        secretKeyRef:
+                          name: ${GLOBAL_NAME}
+                          key: rocket.chat-auth-token-secret
+                    - name: NASA_EARTHDATA_USER
+                      valueFrom:
+                        configMapKeyRef:
+                          name: ${GLOBAL_NAME}
+                          key: env.nasa-earthdata-user
+                    - name: NASA_EARTHDATA_PWD
+                      valueFrom:
+                        secretKeyRef:
+                          name: ${GLOBAL_NAME}
+                          key: nasa-earthdata-pwd
+                  resources:
+                    limits:
+                      cpu: "1"
+                      memory: 1024Mi
+                    requests:
+                      cpu: "500m"
+                      memory: 512Mi
+              restartPolicy: OnFailure