diff --git a/.github/workflows/deployment.yml b/.github/workflows/deployment.yml index 7fd01ad72..afd133fc8 100644 --- a/.github/workflows/deployment.yml +++ b/.github/workflows/deployment.yml @@ -188,6 +188,12 @@ jobs: oc login "${{ secrets.OPENSHIFT_CLUSTER }}" --token="${{ secrets.OC4_DEV_TOKEN }}" PROJ_DEV="e1e498-dev" bash openshift/scripts/oc_provision_noaa_nam_cronjob.sh ${SUFFIX} apply + - name: VIIRS SNOW cronjob + shell: bash + run: | + oc login "${{ secrets.OPENSHIFT_CLUSTER }}" --token="${{ secrets.OC4_DEV_TOKEN }}" + PROJ_DEV="e1e498-dev" bash openshift/scripts/oc_provision_viirs_snow_cronjob.sh ${SUFFIX} apply + # TODO: Delete once pmtiles has run for some time # deploy-tileserv: # name: Deploy tileserv to Dev diff --git a/api/app/jobs/viirs_snow.py b/api/app/jobs/viirs_snow.py new file mode 100644 index 000000000..08c5733da --- /dev/null +++ b/api/app/jobs/viirs_snow.py @@ -0,0 +1,12 @@ +import logging + +from app import configure_logging + +logger = logging.getLogger(__name__) + +def main(): + logger.info("***************Cron job started**************") + +if __name__ == '__main__': + configure_logging() + main() \ No newline at end of file diff --git a/openshift/scripts/oc_provision_viirs_snow_cronjob.sh b/openshift/scripts/oc_provision_viirs_snow_cronjob.sh new file mode 100644 index 000000000..b02800b57 --- /dev/null +++ b/openshift/scripts/oc_provision_viirs_snow_cronjob.sh @@ -0,0 +1,57 @@ +#!/bin/sh -l +# +source "$(dirname ${0})/common/common" + +#% +#% OpenShift Deploy Helper +#% +#% Intended for use with a pull request-based pipeline. +#% Suffixes incl.: pr-###. +#% +#% Usage: +#% +#% ${THIS_FILE} [SUFFIX] [apply] +#% +#% Examples: +#% +#% Provide a PR number. Defaults to a dry-run. +#% ${THIS_FILE} pr-0 +#% +#% Apply when satisfied. +#% ${THIS_FILE} pr-0 apply +#% + +# Target project override for Dev or Prod deployments +# +PROJ_TARGET="${PROJ_TARGET:-${PROJ_DEV}}" + +# Specify a default schedule to run daily at 4am +SCHEDULE="${SCHEDULE:-$((3 + $RANDOM % 54)) 4 * * *}" + +# Process template +OC_PROCESS="oc -n ${PROJ_TARGET} process -f ${TEMPLATE_PATH}/viirs_snow.cronjob.yaml \ +-p JOB_NAME=viirs-snow-${APP_NAME}-${SUFFIX} \ +-p APP_LABEL=${APP_NAME}-${SUFFIX} \ +-p NAME=${APP_NAME} \ +-p SUFFIX=${SUFFIX} \ +-p SCHEDULE=\"${SCHEDULE}\" \ +-p POSTGRES_USER=${POSTGRES_USER:-${APP_NAME}} \ +-p POSTGRES_DATABASE=${POSTGRES_DATABASE:-${APP_NAME}} \ +-p POSTGRES_WRITE_HOST=${POSTGRES_WRITE_HOST:-"patroni-${APP_NAME}-${SUFFIX}-leader"} \ +-p POSTGRES_READ_HOST=${POSTGRES_READ_HOST:-"patroni-${APP_NAME}-${SUFFIX}-replica"} \ +${PROJ_TOOLS:+ "-p PROJ_TOOLS=${PROJ_TOOLS}"} \ +${IMAGE_REGISTRY:+ "-p IMAGE_REGISTRY=${IMAGE_REGISTRY}"}" + +# Apply template (apply or use --dry-run) +# +OC_APPLY="oc -n ${PROJ_TARGET} apply -f -" +[ "${APPLY}" ] || OC_APPLY="${OC_APPLY} --dry-run" + +# Execute commands +# +eval "${OC_PROCESS}" +eval "${OC_PROCESS} | ${OC_APPLY}" + +# Provide oc command instruction +# +display_helper "${OC_PROCESS} | ${OC_APPLY}" diff --git a/openshift/templates/viirs_snow.cronjob.yaml b/openshift/templates/viirs_snow.cronjob.yaml new file mode 100644 index 000000000..20b20dc4c --- /dev/null +++ b/openshift/templates/viirs_snow.cronjob.yaml @@ -0,0 +1,124 @@ +kind: Template +apiVersion: template.openshift.io/v1 +metadata: + name: ${JOB_NAME}-cronjob-template + annotations: + description: "Scheduled task to download and process VIIRS snow data from the Nasa Snow and Ice Data Centre." + tags: "cronjob,viirs,snow" +labels: + app.kubernetes.io/part-of: "${NAME}" + app: ${NAME}-${SUFFIX} +parameters: + - name: NAME + description: Module name + value: wps + - name: GLOBAL_NAME + description: Name of global Module + value: wps-global + - name: SUFFIX + description: Deployment suffix, e.g. pr-### + required: true + - name: PROJ_TOOLS + value: e1e498-tools + - name: JOB_NAME + value: viirs-snow + - name: IMAGE_REGISTRY + required: true + value: image-registry.openshift-image-registry.svc:5000 + - name: POSTGRES_WRITE_HOST + required: true + value: patroni-${NAME}-${SUFFIX}-leader + - name: POSTGRES_READ_HOST + value: patroni-${NAME}-${SUFFIX}-replica + - name: POSTGRES_USER + required: true + value: ${NAME}-${SUFFIX} + - name: POSTGRES_DATABASE + required: true + value: ${NAME}-${SUFFIX} + - name: SCHEDULE + required: true + - name: APP_LABEL + required: true +objects: + - kind: CronJob + apiVersion: batch/v1 + metadata: + name: ${JOB_NAME} + spec: + schedule: ${SCHEDULE} + # We use the "Replace" policy, because we never want the cronjobs to run concurrently, + # and if for whatever reason a cronjob gets stuck, we want the next run to proceed. + # If we were to use Forbid, and a cronjob gets stuck, then we'd stop gathering data until someone + # noticed. We don't want that. + concurrencyPolicy: "Replace" + jobTemplate: + metadata: + labels: + cronjob: ${JOB_NAME} + app: ${APP_LABEL} + spec: + template: + spec: + containers: + - name: ${JOB_NAME} + image: ${IMAGE_REGISTRY}/${PROJ_TOOLS}/${NAME}-api-${SUFFIX}:${SUFFIX} + imagePullPolicy: "Always" + command: + ["poetry", "run", "python", "-m", "app.jobs.viirs_snow"] + env: + - name: POSTGRES_READ_USER + value: ${POSTGRES_USER} + - name: POSTGRES_WRITE_USER + value: ${POSTGRES_USER} + - name: POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + name: ${GLOBAL_NAME} + key: app-db-password + - name: POSTGRES_WRITE_HOST + value: ${POSTGRES_WRITE_HOST} + - name: POSTGRES_READ_HOST + value: ${POSTGRES_READ_HOST} + - name: POSTGRES_PORT + value: "5432" + - name: POSTGRES_DATABASE + value: ${POSTGRES_DATABASE} + - name: ROCKET_URL_POST_MESSAGE + valueFrom: + configMapKeyRef: + name: ${GLOBAL_NAME} + key: rocket.chat-url-post-message + - name: ROCKET_CHANNEL + valueFrom: + configMapKeyRef: + name: ${GLOBAL_NAME} + key: rocket.chat-channel + - name: ROCKET_USER_ID + valueFrom: + secretKeyRef: + name: ${GLOBAL_NAME} + key: rocket.chat-user-id-secret + - name: ROCKET_AUTH_TOKEN + valueFrom: + secretKeyRef: + name: ${GLOBAL_NAME} + key: rocket.chat-auth-token-secret + - name: NASA_EARTHDATA_USER + valueFrom: + configMapKeyRef: + name: ${GLOBAL_NAME} + key: env.nasa-earthdata-user + - name: NASA_EARTHDATA_PWD + valueFrom: + secretKeyRef: + name: ${GLOBAL_NAME} + key: nasa-earthdata-pwd + resources: + limits: + cpu: "1" + memory: 1024Mi + requests: + cpu: "500m" + memory: 512Mi + restartPolicy: OnFailure