Add update_monitors.sh script and readme for datadog migration (#13)

* Add update_monitors.sh script and readme for datadog migration

* Add and document datadog-webhook.tf

* Add nice set commands

* Add continue prompt with list of monitor names

* Fix prompt to default No

dd_migration/Readme.md

@@ -0,0 +1,19 @@
+# Datadog Migration Tools
+
+Scripts and terraform files to aid in migration for customers using datadog for alerting
+
+## Tools
+
+## `update_monitors.sh`
+
+This script will search a datadog instance for all monitors with a given notification method (such as `@slack-platform-alerts`) and create a new monitor with the same name and all the same settings, but with a new notification method (such as `@firehydrant-platform-alerts`).  Optionally, the `UPDATE_IN_PLACE` variable in the script can be set to `true`, in which case the existing monitor will be modified to use the new notification method instead.
+
+This script requires that a datadog API key be exported as `DD_API_KEY`, as well as an application key exported as `DD_APP_KEY`.  Note that the application key must have all scopes needed for modifying all expected alerts and so should be created by someone with admin access.  In the script, the datadog site parameter `DD_SITE_PARAMETER` needs to be set to connect with any site other than the default US1.  See this page for the appropriate value: https://docs.datadoghq.com/getting_started/site/#access-the-datadog-site
+
+Known issue: All monitors in question need to have some text (apart from the notification method) in the `message` field for the monitor.  Datadog allows this field to contain the notification method only, but then returns an empty string for the message as part of the API and thus it can't be modified in this way.
+
+## `datadog-webhook.tf`
+
+This is an example terraform file for creating a signals compatible webhook at datadog.  It requires the API key and Application key as described above, as well as the base datadog API url as described in the provider setup here: https://registry.terraform.io/providers/DataDog/datadog/latest/docs
+
+The team-specific URL would be added where indicated.  This would create a notification method addressable in datadog as `@webhook-firehydrant-myteam`

dd_migration/datadog-webhook.tf

@@ -0,0 +1,33 @@
+terraform {
+  required_version = "~> 1.5.5"
+
+  required_providers {
+    datadog = {
+      source  = "DataDog/datadog"
+      version = ">= 3.38.0"
+    }
+  }
+}
+
+provider "datadog" {
+  api_key = ""
+  app_key = ""
+  api_url = ""
+}
+
+resource "datadog_webhook" "firehydrant-myteam" {
+  name           = "firehydrant-myteam"
+  url            = "<team URL>"
+  encode_as      = "json"
+  payload        = <<EOF
+{
+    "summary": "$EVENT_TITLE",
+    "body": "$EVENT_MSG",
+    "unique_key": "$ALERT_ID",
+    "level": "$ALERT_TYPE",
+    "status": "$ALERT_TRANSITION",
+    "links": [{"href": "$LINK", "text": "Datadog Monitor"}],
+    "images": [{"src": "$SNAPSHOT", "alt": "Snapshot from $EVENT_TITLE"}]
+}
+EOF
+}

dd_migration/update_monitors.sh

@@ -0,0 +1,99 @@
+#! /usr/bin/env bash
+set -eo pipefail
+
+# By default, we will make a copy of any alert with notification OLD_NOTIFICATION that is identical in all other ways but replaces OLD_NOTIFICATION
+# with NEW_NOTIFICATION.  Setting UPDATE_IN_PLACE here changes that behavior to modify the existing alert instead of making a new one.
+UPDATE_IN_PLACE="false"
+# The existing notification as it appears in the Datadog message.  For example @slack-platform-alerts
+OLD_NOTIFICATION=""
+# The new notification as it would appear in the Datadog message.  For example @firehydrant-platform-alerts
+NEW_NOTIFICATION=""
+# The appropriate Datadog site parameter from the table here: https://docs.datadoghq.com/getting_started/site/#access-the-datadog-site
+DD_SITE_PARAMETER="datadoghq.com"
+
+if [ -z "$DD_API_KEY" ]
+then
+  echo "Please export your datadog API key as DD_API_KEY"
+  exit 1
+fi
+
+if [ -z "$DD_APP_KEY" ]
+then
+  echo "Please export your datadog application key as DD_APP_KEY"
+  exit 1
+fi
+
+set -u
+
+URL_ENCODED_OLD_NOTIFICATION=$(echo ${OLD_NOTIFICATION:1}| jq -Rr @uri)
+
+RAW_SEARCH=$(curl -s --fail-with-body 'https://api.'"$DD_SITE_PARAMETER"'/api/v1/monitor/search?query=notification%3A"'"$URL_ENCODED_OLD_NOTIFICATION"'"' -H "Accept: application/json" -H "DD-API-KEY: $DD_API_KEY" -H "DD-APPLICATION-KEY: $DD_APP_KEY")
+
+PAGES=$(echo $RAW_SEARCH | jq .metadata.page_count)
+MONITOR_COUNT=$(echo $RAW_SEARCH | jq .metadata.total_count)
+
+MONITORS=()
+for ((i=0; i<$PAGES; i++)); do
+  MONITORS+=$(curl -s --fail-with-body 'https://api.'"$DD_SITE_PARAMETER"'/api/v1/monitor/search?query=notification%3A"'"$URL_ENCODED_OLD_NOTIFICATION"'"&page='"$i" -H "Accept: application/json" -H "DD-API-KEY: $DD_API_KEY" -H "DD-APPLICATION-KEY: $DD_APP_KEY" | jq '.monitors | .[] | .id')
+  MONITORS+=" "
+done
+
+echo "Found $MONITOR_COUNT monitors..."
+
+monitornames=()
+for monitorid in $MONITORS; do
+  rawmonitor=$(curl -s --fail-with-body "https://api.$DD_SITE_PARAMETER/api/v1/monitor/$monitorid" -H "Accept: application/json" -H "DD-API-KEY: $DD_API_KEY" -H "DD-APPLICATION-KEY: $DD_APP_KEY")
+  monitornames+=$(echo $rawmonitor | jq .name)
+  monitornames+='\n'
+done
+
+if [ $UPDATE_IN_PLACE == "true" ]
+then
+  echo -e "The following monitors will be updated to point to $NEW_NOTIFICATION:\n"
+else
+  echo -e "Copies of the following monitors will be created pointing to $NEW_NOTIFICATION:\n"
+fi
+
+echo -e $monitornames
+
+read -p "Type Yes to continue: " yn
+case $yn in
+    Yes ) ;;
+    * ) exit 1;;
+esac
+
+for monitorid in $MONITORS; do
+  rawmonitor=$(curl -s --fail-with-body "https://api.$DD_SITE_PARAMETER/api/v1/monitor/$monitorid" -H "Accept: application/json" -H "DD-API-KEY: $DD_API_KEY" -H "DD-APPLICATION-KEY: $DD_APP_KEY")
+  new_message=$(echo $rawmonitor | jq .message | sed "s/$OLD_NOTIFICATION/$NEW_NOTIFICATION/g")
+  if [ $UPDATE_IN_PLACE == "true" ]
+  then
+    curl -X PUT -s --fail-with-body "https://api.$DD_SITE_PARAMETER/api/v1/monitor/$monitorid" \
+    -H "Accept: application/json" \
+    -H "Content-Type: application/json" \
+    -H "DD-API-KEY: $DD_API_KEY" \
+    -H "DD-APPLICATION-KEY: $DD_APP_KEY" \
+    -d { "message": $new_message }
+
+    echo "updated $(echo $rawmonitor | jq .name) in place"
+  else
+    payload='{
+  "name": '"$(echo $rawmonitor | jq .name)"',
+  "message": '"$new_message"',
+  "options": '"$(echo $rawmonitor | jq .options)"',
+  "priority": '"$(echo $rawmonitor | jq .priority)"',
+  "query": '"$(echo $rawmonitor | jq .query)"',
+  "restricted_roles": '"$(echo $rawmonitor | jq .restricted_roles)"',
+  "tags": '"$(echo $rawmonitor | jq .tags)"',
+  "type": '"$(echo $rawmonitor | jq .type)"'
+}'
+
+    curl -X POST -s --fail-with-body "https://api.$DD_SITE_PARAMETER/api/v1/monitor" \
+    -H "Accept: application/json" \
+    -H "Content-Type: application/json" \
+    -H "DD-API-KEY: $DD_API_KEY" \
+    -H "DD-APPLICATION-KEY: $DD_APP_KEY" \
+    -d "$payload"
+
+    echo "created new copy of $(echo $rawmonitor | jq .name) with notifications targeting $NEW_NOTIFICATION"
+  fi
+done