From ae14191df4e93908fe1d7cecf02e9b800ff7c50c Mon Sep 17 00:00:00 2001 From: Stephen Soltesz Date: Mon, 4 Jun 2018 13:27:05 -0400 Subject: [PATCH 1/4] Add a github issue search link to slack notifications --- apply-global-prometheus.sh | 5 +++++ config/federation/alertmanager/config.yml.template | 2 ++ 2 files changed, 7 insertions(+) diff --git a/apply-global-prometheus.sh b/apply-global-prometheus.sh index 8202be4e..aa5e9056 100755 --- a/apply-global-prometheus.sh +++ b/apply-global-prometheus.sh @@ -112,6 +112,7 @@ kubectl create configmap grafana-env \ # Note: Only enable the github reciever for the production project: mlab-oti. SLACK_CHANNEL_URL_NAME=AM_SLACK_CHANNEL_URL_${PROJECT/-/_} GITHUB_RECEIVER_URL= +GITHUB_ISSUE_QUERY= SHORT_PROJECT=${PROJECT/mlab-/} if [[ ${PROJECT} = "mlab-oti" ]] ; then @@ -119,6 +120,9 @@ if [[ ${PROJECT} = "mlab-oti" ]] ; then kubectl create secret generic github-secrets \ "--from-literal=auth-token=${GITHUB_RECEIVER_AUTH_TOKEN}" \ --dry-run -o json | kubectl apply -f - + + # For production, annotate slack messages with a link to view open alert issues. + GITHUB_ISSUE_QUERY="https://github.com/issues?q=is%3Aissue+user%3Am-lab+author%3Ameasurementlab+is%3Aopen" fi # Note: without a url, alertmanager will fail to start. But, for non-production @@ -128,6 +132,7 @@ GITHUB_RECEIVER_URL=http://github-receiver-service.default.svc.cluster.local:939 sed -e 's|{{SLACK_CHANNEL_URL}}|'${!SLACK_CHANNEL_URL_NAME}'|g' \ -e 's|{{GITHUB_RECEIVER_URL}}|'$GITHUB_RECEIVER_URL'|g' \ -e 's|{{SHORT_PROJECT}}|'$SHORT_PROJECT'|g' \ + -e 's|{{GITHUB_ISSUE_QUERY}}|'$GITHUB_ISSUE_QUERY'|g' \ config/federation/alertmanager/config.yml.template > \ config/federation/alertmanager/config.yml diff --git a/config/federation/alertmanager/config.yml.template b/config/federation/alertmanager/config.yml.template index 71dbfaa3..6ad17948 100644 --- a/config/federation/alertmanager/config.yml.template +++ b/config/federation/alertmanager/config.yml.template @@ -83,6 +83,7 @@ receivers: api_url: {{SLACK_CHANNEL_URL}} channel: alerts-{{SHORT_PROJECT}} username: alert-page + text: '{{GITHUB_ISSUE_QUERY}}' webhook_configs: - url: '{{GITHUB_RECEIVER_URL}}' @@ -92,5 +93,6 @@ receivers: api_url: {{SLACK_CHANNEL_URL}} channel: alerts-{{SHORT_PROJECT}} username: alert-email + text: '{{GITHUB_ISSUE_QUERY}}' webhook_configs: - url: '{{GITHUB_RECEIVER_URL}}' From 91fe5b9a7c422ec73f913a072d4aa909e9e6cf8a Mon Sep 17 00:00:00 2001 From: Stephen Soltesz Date: Mon, 4 Jun 2018 13:30:26 -0400 Subject: [PATCH 2/4] apply to sandbox --- apply-global-prometheus.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/apply-global-prometheus.sh b/apply-global-prometheus.sh index aa5e9056..15272ede 100755 --- a/apply-global-prometheus.sh +++ b/apply-global-prometheus.sh @@ -124,6 +124,8 @@ if [[ ${PROJECT} = "mlab-oti" ]] ; then # For production, annotate slack messages with a link to view open alert issues. GITHUB_ISSUE_QUERY="https://github.com/issues?q=is%3Aissue+user%3Am-lab+author%3Ameasurementlab+is%3Aopen" fi +# TODO: remote after testing sandbox. +GITHUB_ISSUE_QUERY="https://github.com/issues?q=is%3Aissue+user%3Am-lab+author%3Ameasurementlab+is%3Aopen" # Note: without a url, alertmanager will fail to start. But, for non-production # projects, there will be no github receiver running. This should be a no-op. From 99240a064864c32bf787855ca1f9c04156119678 Mon Sep 17 00:00:00 2001 From: Stephen Soltesz Date: Mon, 4 Jun 2018 13:40:18 -0400 Subject: [PATCH 3/4] Fix whitespace --- config/federation/alertmanager/config.yml.template | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config/federation/alertmanager/config.yml.template b/config/federation/alertmanager/config.yml.template index 6ad17948..a7776887 100644 --- a/config/federation/alertmanager/config.yml.template +++ b/config/federation/alertmanager/config.yml.template @@ -83,7 +83,7 @@ receivers: api_url: {{SLACK_CHANNEL_URL}} channel: alerts-{{SHORT_PROJECT}} username: alert-page - text: '{{GITHUB_ISSUE_QUERY}}' + text: '{{GITHUB_ISSUE_QUERY}}' webhook_configs: - url: '{{GITHUB_RECEIVER_URL}}' @@ -93,6 +93,6 @@ receivers: api_url: {{SLACK_CHANNEL_URL}} channel: alerts-{{SHORT_PROJECT}} username: alert-email - text: '{{GITHUB_ISSUE_QUERY}}' + text: '{{GITHUB_ISSUE_QUERY}}' webhook_configs: - url: '{{GITHUB_RECEIVER_URL}}' From 71f879c130b34490915620bce1dd4e6946f3789a Mon Sep 17 00:00:00 2001 From: Stephen Soltesz Date: Mon, 4 Jun 2018 16:31:40 -0400 Subject: [PATCH 4/4] Restore sandbox config --- apply-global-prometheus.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/apply-global-prometheus.sh b/apply-global-prometheus.sh index 15272ede..aa5e9056 100755 --- a/apply-global-prometheus.sh +++ b/apply-global-prometheus.sh @@ -124,8 +124,6 @@ if [[ ${PROJECT} = "mlab-oti" ]] ; then # For production, annotate slack messages with a link to view open alert issues. GITHUB_ISSUE_QUERY="https://github.com/issues?q=is%3Aissue+user%3Am-lab+author%3Ameasurementlab+is%3Aopen" fi -# TODO: remote after testing sandbox. -GITHUB_ISSUE_QUERY="https://github.com/issues?q=is%3Aissue+user%3Am-lab+author%3Ameasurementlab+is%3Aopen" # Note: without a url, alertmanager will fail to start. But, for non-production # projects, there will be no github receiver running. This should be a no-op.