Skip to content

Commit

Permalink
Snow processing devops prep (#3310)
Browse files Browse the repository at this point in the history
  • Loading branch information
dgboss authored Jan 11, 2024
1 parent 2300e96 commit 5ef15bf
Show file tree
Hide file tree
Showing 4 changed files with 199 additions and 0 deletions.
6 changes: 6 additions & 0 deletions .github/workflows/deployment.yml
Original file line number Diff line number Diff line change
Expand Up @@ -188,6 +188,12 @@ jobs:
oc login "${{ secrets.OPENSHIFT_CLUSTER }}" --token="${{ secrets.OC4_DEV_TOKEN }}"
PROJ_DEV="e1e498-dev" bash openshift/scripts/oc_provision_noaa_nam_cronjob.sh ${SUFFIX} apply
- name: VIIRS SNOW cronjob
shell: bash
run: |
oc login "${{ secrets.OPENSHIFT_CLUSTER }}" --token="${{ secrets.OC4_DEV_TOKEN }}"
PROJ_DEV="e1e498-dev" bash openshift/scripts/oc_provision_viirs_snow_cronjob.sh ${SUFFIX} apply
# TODO: Delete once pmtiles has run for some time
# deploy-tileserv:
# name: Deploy tileserv to Dev
Expand Down
12 changes: 12 additions & 0 deletions api/app/jobs/viirs_snow.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
import logging

from app import configure_logging

logger = logging.getLogger(__name__)

def main():
logger.info("***************Cron job started**************")

if __name__ == '__main__':
configure_logging()
main()
57 changes: 57 additions & 0 deletions openshift/scripts/oc_provision_viirs_snow_cronjob.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,57 @@
#!/bin/sh -l
#
source "$(dirname ${0})/common/common"

#%
#% OpenShift Deploy Helper
#%
#% Intended for use with a pull request-based pipeline.
#% Suffixes incl.: pr-###.
#%
#% Usage:
#%
#% ${THIS_FILE} [SUFFIX] [apply]
#%
#% Examples:
#%
#% Provide a PR number. Defaults to a dry-run.
#% ${THIS_FILE} pr-0
#%
#% Apply when satisfied.
#% ${THIS_FILE} pr-0 apply
#%

# Target project override for Dev or Prod deployments
#
PROJ_TARGET="${PROJ_TARGET:-${PROJ_DEV}}"

# Specify a default schedule to run daily at 4am
SCHEDULE="${SCHEDULE:-$((3 + $RANDOM % 54)) 4 * * *}"

# Process template
OC_PROCESS="oc -n ${PROJ_TARGET} process -f ${TEMPLATE_PATH}/viirs_snow.cronjob.yaml \
-p JOB_NAME=viirs-snow-${APP_NAME}-${SUFFIX} \
-p APP_LABEL=${APP_NAME}-${SUFFIX} \
-p NAME=${APP_NAME} \
-p SUFFIX=${SUFFIX} \
-p SCHEDULE=\"${SCHEDULE}\" \
-p POSTGRES_USER=${POSTGRES_USER:-${APP_NAME}} \
-p POSTGRES_DATABASE=${POSTGRES_DATABASE:-${APP_NAME}} \
-p POSTGRES_WRITE_HOST=${POSTGRES_WRITE_HOST:-"patroni-${APP_NAME}-${SUFFIX}-leader"} \
-p POSTGRES_READ_HOST=${POSTGRES_READ_HOST:-"patroni-${APP_NAME}-${SUFFIX}-replica"} \
${PROJ_TOOLS:+ "-p PROJ_TOOLS=${PROJ_TOOLS}"} \
${IMAGE_REGISTRY:+ "-p IMAGE_REGISTRY=${IMAGE_REGISTRY}"}"

# Apply template (apply or use --dry-run)
#
OC_APPLY="oc -n ${PROJ_TARGET} apply -f -"
[ "${APPLY}" ] || OC_APPLY="${OC_APPLY} --dry-run"

# Execute commands
#
eval "${OC_PROCESS}"
eval "${OC_PROCESS} | ${OC_APPLY}"

# Provide oc command instruction
#
display_helper "${OC_PROCESS} | ${OC_APPLY}"
124 changes: 124 additions & 0 deletions openshift/templates/viirs_snow.cronjob.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,124 @@
kind: Template
apiVersion: template.openshift.io/v1
metadata:
name: ${JOB_NAME}-cronjob-template
annotations:
description: "Scheduled task to download and process VIIRS snow data from the Nasa Snow and Ice Data Centre."
tags: "cronjob,viirs,snow"
labels:
app.kubernetes.io/part-of: "${NAME}"
app: ${NAME}-${SUFFIX}
parameters:
- name: NAME
description: Module name
value: wps
- name: GLOBAL_NAME
description: Name of global Module
value: wps-global
- name: SUFFIX
description: Deployment suffix, e.g. pr-###
required: true
- name: PROJ_TOOLS
value: e1e498-tools
- name: JOB_NAME
value: viirs-snow
- name: IMAGE_REGISTRY
required: true
value: image-registry.openshift-image-registry.svc:5000
- name: POSTGRES_WRITE_HOST
required: true
value: patroni-${NAME}-${SUFFIX}-leader
- name: POSTGRES_READ_HOST
value: patroni-${NAME}-${SUFFIX}-replica
- name: POSTGRES_USER
required: true
value: ${NAME}-${SUFFIX}
- name: POSTGRES_DATABASE
required: true
value: ${NAME}-${SUFFIX}
- name: SCHEDULE
required: true
- name: APP_LABEL
required: true
objects:
- kind: CronJob
apiVersion: batch/v1
metadata:
name: ${JOB_NAME}
spec:
schedule: ${SCHEDULE}
# We use the "Replace" policy, because we never want the cronjobs to run concurrently,
# and if for whatever reason a cronjob gets stuck, we want the next run to proceed.
# If we were to use Forbid, and a cronjob gets stuck, then we'd stop gathering data until someone
# noticed. We don't want that.
concurrencyPolicy: "Replace"
jobTemplate:
metadata:
labels:
cronjob: ${JOB_NAME}
app: ${APP_LABEL}
spec:
template:
spec:
containers:
- name: ${JOB_NAME}
image: ${IMAGE_REGISTRY}/${PROJ_TOOLS}/${NAME}-api-${SUFFIX}:${SUFFIX}
imagePullPolicy: "Always"
command:
["poetry", "run", "python", "-m", "app.jobs.viirs_snow"]
env:
- name: POSTGRES_READ_USER
value: ${POSTGRES_USER}
- name: POSTGRES_WRITE_USER
value: ${POSTGRES_USER}
- name: POSTGRES_PASSWORD
valueFrom:
secretKeyRef:
name: ${GLOBAL_NAME}
key: app-db-password
- name: POSTGRES_WRITE_HOST
value: ${POSTGRES_WRITE_HOST}
- name: POSTGRES_READ_HOST
value: ${POSTGRES_READ_HOST}
- name: POSTGRES_PORT
value: "5432"
- name: POSTGRES_DATABASE
value: ${POSTGRES_DATABASE}
- name: ROCKET_URL_POST_MESSAGE
valueFrom:
configMapKeyRef:
name: ${GLOBAL_NAME}
key: rocket.chat-url-post-message
- name: ROCKET_CHANNEL
valueFrom:
configMapKeyRef:
name: ${GLOBAL_NAME}
key: rocket.chat-channel
- name: ROCKET_USER_ID
valueFrom:
secretKeyRef:
name: ${GLOBAL_NAME}
key: rocket.chat-user-id-secret
- name: ROCKET_AUTH_TOKEN
valueFrom:
secretKeyRef:
name: ${GLOBAL_NAME}
key: rocket.chat-auth-token-secret
- name: NASA_EARTHDATA_USER
valueFrom:
configMapKeyRef:
name: ${GLOBAL_NAME}
key: env.nasa-earthdata-user
- name: NASA_EARTHDATA_PWD
valueFrom:
secretKeyRef:
name: ${GLOBAL_NAME}
key: nasa-earthdata-pwd
resources:
limits:
cpu: "1"
memory: 1024Mi
requests:
cpu: "500m"
memory: 512Mi
restartPolicy: OnFailure

0 comments on commit 5ef15bf

Please sign in to comment.