From d2ef798953f67f3e4ff36f1761fd5befedbe091e Mon Sep 17 00:00:00 2001 From: Jackson Sabey Date: Fri, 30 Aug 2024 23:30:13 -0700 Subject: [PATCH 1/8] repeat-interval --- README.md | 19 +++++++++-------- alerting/alert/alert.go | 4 ++++ config/endpoint/endpoint.go | 3 +++ watchdog/alerting.go | 22 +++++++++++++++---- watchdog/alerting_test.go | 42 +++++++++++++++++++++++++++++++++++++ 5 files changed, 77 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 9b3d59106..880cf2fcc 100644 --- a/README.md +++ b/README.md @@ -520,15 +520,16 @@ individual endpoints with configurable descriptions and thresholds. Alerts are configured at the endpoint level like so: -| Parameter | Description | Default | -|:-----------------------------|:-------------------------------------------------------------------------------|:--------------| -| `alerts` | List of all alerts for a given endpoint. | `[]` | -| `alerts[].type` | Type of alert.
See table below for all valid types. | Required `""` | -| `alerts[].enabled` | Whether to enable the alert. | `true` | -| `alerts[].failure-threshold` | Number of failures in a row needed before triggering the alert. | `3` | -| `alerts[].success-threshold` | Number of successes in a row before an ongoing incident is marked as resolved. | `2` | -| `alerts[].send-on-resolved` | Whether to send a notification once a triggered alert is marked as resolved. | `false` | -| `alerts[].description` | Description of the alert. Will be included in the alert sent. | `""` | +| Parameter | Description | Default | +|:----------------------------------|:-------------------------------------------------------------------------------|:--------------| +| `alerts` | List of all alerts for a given endpoint. | `[]` | +| `alerts[].type` | Type of alert.
See table below for all valid types. | Required `""` | +| `alerts[].enabled` | Whether to enable the alert. | `true` | +| `alerts[].failure-threshold` | Number of failures in a row needed before triggering the alert. | `3` | +| `alerts[].success-threshold` | Number of successes in a row before an ongoing incident is marked as resolved. | `2` | +| `alerts[].send-on-resolved` | Whether to send a notification once a triggered alert is marked as resolved. | `false` | +| `alerts[].description` | Description of the alert. Will be included in the alert sent. | `""` | +| `alerts[].repeat-interval` | Configuration for setting an interval between reminders. | `""` | Here's an example of what an alert configuration might look like at the endpoint level: ```yaml diff --git a/alerting/alert/alert.go b/alerting/alert/alert.go index 2afcd8b50..b35a18188 100644 --- a/alerting/alert/alert.go +++ b/alerting/alert/alert.go @@ -6,6 +6,7 @@ import ( "errors" "strconv" "strings" + "time" ) var ( @@ -48,6 +49,9 @@ type Alert struct { // ongoing/triggered incidents ResolveKey string `yaml:"-"` + // RepeatInterval is the interval between reminders + RepeatInterval time.Duration `yaml:"repeat-interval,omitempty"` + // Triggered is used to determine whether an alert has been triggered. When an alert is resolved, this value // should be set back to false. It is used to prevent the same alert from going out twice. // diff --git a/config/endpoint/endpoint.go b/config/endpoint/endpoint.go index ac765c1a5..e73d40758 100644 --- a/config/endpoint/endpoint.go +++ b/config/endpoint/endpoint.go @@ -121,6 +121,9 @@ type Endpoint struct { // NumberOfSuccessesInARow is the number of successful evaluations in a row NumberOfSuccessesInARow int `yaml:"-"` + + // LastReminderSent is the time at which the last reminder was sent for this endpoint. + LastReminderSent time.Time `yaml:"-"` } // IsEnabled returns whether the endpoint is enabled or not diff --git a/watchdog/alerting.go b/watchdog/alerting.go index 6fd7a2468..7f6eb0921 100644 --- a/watchdog/alerting.go +++ b/watchdog/alerting.go @@ -4,6 +4,7 @@ import ( "errors" "log" "os" + "time" "github.com/TwiN/gatus/v5/alerting" "github.com/TwiN/gatus/v5/config/endpoint" @@ -30,16 +31,25 @@ func handleAlertsToTrigger(ep *endpoint.Endpoint, result *endpoint.Result, alert if !endpointAlert.IsEnabled() || endpointAlert.FailureThreshold > ep.NumberOfFailuresInARow { continue } - if endpointAlert.Triggered { + // Determine if an initial alert should be sent + sendInitialAlert := !endpointAlert.Triggered + // Determine if a reminder should be sent + sendReminder := endpointAlert.Triggered && endpointAlert.RepeatInterval > 0 && time.Since(ep.LastReminderSent) >= endpointAlert.RepeatInterval + // If neither initial alert nor reminder needs to be sent, skip to the next alert + if !sendInitialAlert && !sendReminder { if debug { - log.Printf("[watchdog.handleAlertsToTrigger] Alert for endpoint=%s with description='%s' has already been TRIGGERED, skipping", ep.Name, endpointAlert.GetDescription()) + log.Printf("[watchdog.handleAlertsToTrigger] Alert for endpoint=%s with description='%s' is not due for triggering or reminding, skipping", ep.Name, endpointAlert.GetDescription()) } continue } alertProvider := alertingConfig.GetAlertingProviderByAlertType(endpointAlert.Type) if alertProvider != nil { - log.Printf("[watchdog.handleAlertsToTrigger] Sending %s alert because alert for endpoint=%s with description='%s' has been TRIGGERED", endpointAlert.Type, ep.Name, endpointAlert.GetDescription()) var err error + alertType := "reminder" + if sendInitialAlert { + alertType = "initial" + } + log.Printf("[watchdog.handleAlertsToTrigger] Sending %s %s alert because alert for endpoint=%s with description='%s' has been TRIGGERED", alertType, endpointAlert.Type, ep.Name, endpointAlert.GetDescription()) if os.Getenv("MOCK_ALERT_PROVIDER") == "true" { if os.Getenv("MOCK_ALERT_PROVIDER_ERROR") == "true" { err = errors.New("error") @@ -50,7 +60,11 @@ func handleAlertsToTrigger(ep *endpoint.Endpoint, result *endpoint.Result, alert if err != nil { log.Printf("[watchdog.handleAlertsToTrigger] Failed to send an alert for endpoint=%s: %s", ep.Name, err.Error()) } else { - endpointAlert.Triggered = true + // Mark initial alert as triggered and update last reminder time + if sendInitialAlert { + endpointAlert.Triggered = true + } + ep.LastReminderSent = time.Now() if err := store.Get().UpsertTriggeredEndpointAlert(ep, endpointAlert); err != nil { log.Printf("[watchdog.handleAlertsToTrigger] Failed to persist triggered endpoint alert for endpoint with key=%s: %s", ep.Key(), err.Error()) } diff --git a/watchdog/alerting_test.go b/watchdog/alerting_test.go index 914355e46..8b0a61491 100644 --- a/watchdog/alerting_test.go +++ b/watchdog/alerting_test.go @@ -3,6 +3,7 @@ package watchdog import ( "os" "testing" + "time" "github.com/TwiN/gatus/v5/alerting" "github.com/TwiN/gatus/v5/alerting/alert" @@ -487,6 +488,47 @@ func TestHandleAlertingWithProviderThatOnlyReturnsErrorOnResolve(t *testing.T) { verify(t, ep, 0, 2, false, "") } +func TestHandleAlertingWithRepeatInterval(t *testing.T) { + _ = os.Setenv("MOCK_ALERT_PROVIDER", "true") + defer os.Clearenv() + + cfg := &config.Config{ + Debug: true, + Alerting: &alerting.Config{ + Custom: &custom.AlertProvider{ + URL: "https://twin.sh/health", + Method: "GET", + }, + }, + } + enabled := true + ep := &endpoint.Endpoint{ + URL: "https://example.com", + Alerts: []*alert.Alert{ + { + Type: alert.TypeCustom, + Enabled: &enabled, + FailureThreshold: 2, + SuccessThreshold: 3, + SendOnResolved: &enabled, + Triggered: false, + RepeatInterval: 1 * time.Second, + }, + }, + } + + verify(t, ep, 0, 0, false, "The alert shouldn't start triggered") + HandleAlerting(ep, &endpoint.Result{Success: false}, cfg.Alerting, cfg.Debug) + verify(t, ep, 1, 0, false, "The alert shouldn't have triggered") + HandleAlerting(ep, &endpoint.Result{Success: false}, cfg.Alerting, cfg.Debug) + verify(t, ep, 2, 0, true, "The alert should've triggered") + HandleAlerting(ep, &endpoint.Result{Success: false}, cfg.Alerting, cfg.Debug) + verify(t, ep, 3, 0, true, "The alert should still be triggered") + HandleAlerting(ep, &endpoint.Result{Success: false}, cfg.Alerting, cfg.Debug) + verify(t, ep, 4, 0, true, "The alert should still be triggered") + HandleAlerting(ep, &endpoint.Result{Success: true}, cfg.Alerting, cfg.Debug) +} + func verify(t *testing.T, ep *endpoint.Endpoint, expectedNumberOfFailuresInARow, expectedNumberOfSuccessInARow int, expectedTriggered bool, expectedTriggeredReason string) { if ep.NumberOfFailuresInARow != expectedNumberOfFailuresInARow { t.Errorf("endpoint.NumberOfFailuresInARow should've been %d, got %d", expectedNumberOfFailuresInARow, ep.NumberOfFailuresInARow) From e03fe101469513b35ebec658a25680ed67a8b85f Mon Sep 17 00:00:00 2001 From: Jackson Sabey Date: Sat, 31 Aug 2024 01:16:54 -0700 Subject: [PATCH 2/8] set default alert repeat interval --- alerting/provider/provider.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/alerting/provider/provider.go b/alerting/provider/provider.go index 30e805e47..862ddcc2f 100644 --- a/alerting/provider/provider.go +++ b/alerting/provider/provider.go @@ -57,6 +57,9 @@ func ParseWithDefaultAlert(providerDefaultAlert, endpointAlert *alert.Alert) { if endpointAlert.SuccessThreshold == 0 { endpointAlert.SuccessThreshold = providerDefaultAlert.SuccessThreshold } + if endpointAlert.RepeatInterval > 0 { + endpointAlert.RepeatInterval = providerDefaultAlert.RepeatInterval + } } var ( From a789c006da3ef2d13514ccbc0ef7e10c44f39521 Mon Sep 17 00:00:00 2001 From: Jackson Sabey Date: Sat, 31 Aug 2024 14:29:38 -0700 Subject: [PATCH 3/8] check if last reminder is zero, set last reminder on resolved --- watchdog/alerting.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/watchdog/alerting.go b/watchdog/alerting.go index 7f6eb0921..243ce914a 100644 --- a/watchdog/alerting.go +++ b/watchdog/alerting.go @@ -34,7 +34,8 @@ func handleAlertsToTrigger(ep *endpoint.Endpoint, result *endpoint.Result, alert // Determine if an initial alert should be sent sendInitialAlert := !endpointAlert.Triggered // Determine if a reminder should be sent - sendReminder := endpointAlert.Triggered && endpointAlert.RepeatInterval > 0 && time.Since(ep.LastReminderSent) >= endpointAlert.RepeatInterval + sendReminder := endpointAlert.Triggered && endpointAlert.RepeatInterval > 0 && + (ep.LastReminderSent.IsZero() || time.Since(ep.LastReminderSent) >= endpointAlert.RepeatInterval) // If neither initial alert nor reminder needs to be sent, skip to the next alert if !sendInitialAlert && !sendReminder { if debug { @@ -109,4 +110,5 @@ func handleAlertsToResolve(ep *endpoint.Endpoint, result *endpoint.Result, alert } } ep.NumberOfFailuresInARow = 0 + ep.LastReminderSent = time.Now() } From f83ce9993fbed63da201e8f6cffda1289fcd2fbd Mon Sep 17 00:00:00 2001 From: Jackson Sabey Date: Sat, 31 Aug 2024 14:52:39 -0700 Subject: [PATCH 4/8] better debugging --- watchdog/alerting.go | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/watchdog/alerting.go b/watchdog/alerting.go index 243ce914a..c57a0f7b3 100644 --- a/watchdog/alerting.go +++ b/watchdog/alerting.go @@ -34,12 +34,16 @@ func handleAlertsToTrigger(ep *endpoint.Endpoint, result *endpoint.Result, alert // Determine if an initial alert should be sent sendInitialAlert := !endpointAlert.Triggered // Determine if a reminder should be sent - sendReminder := endpointAlert.Triggered && endpointAlert.RepeatInterval > 0 && - (ep.LastReminderSent.IsZero() || time.Since(ep.LastReminderSent) >= endpointAlert.RepeatInterval) + var lastReminder time.Duration + if !ep.LastReminderSent.IsZero() { + lastReminder = time.Since(ep.LastReminderSent) + } + sendReminder := endpointAlert.Triggered && endpointAlert.RepeatInterval > 0 && lastReminder >= endpointAlert.RepeatInterval // If neither initial alert nor reminder needs to be sent, skip to the next alert if !sendInitialAlert && !sendReminder { if debug { - log.Printf("[watchdog.handleAlertsToTrigger] Alert for endpoint=%s with description='%s' is not due for triggering or reminding, skipping", ep.Name, endpointAlert.GetDescription()) + log.Printf("[watchdog.handleAlertsToTrigger] Alert for endpoint=%s with description='%s' is not due for triggering or reminding (interval: %s last: %s), skipping", + ep.Name, endpointAlert.GetDescription(), endpointAlert.RepeatInterval, lastReminder) } continue } From f52432aa4bd4bb110aced4e6e6928a268224c300 Mon Sep 17 00:00:00 2001 From: Jackson Sabey Date: Sat, 31 Aug 2024 15:07:36 -0700 Subject: [PATCH 5/8] fix default --- alerting/provider/provider.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/alerting/provider/provider.go b/alerting/provider/provider.go index 862ddcc2f..230f73cc0 100644 --- a/alerting/provider/provider.go +++ b/alerting/provider/provider.go @@ -57,7 +57,7 @@ func ParseWithDefaultAlert(providerDefaultAlert, endpointAlert *alert.Alert) { if endpointAlert.SuccessThreshold == 0 { endpointAlert.SuccessThreshold = providerDefaultAlert.SuccessThreshold } - if endpointAlert.RepeatInterval > 0 { + if endpointAlert.RepeatInterval == 0 { endpointAlert.RepeatInterval = providerDefaultAlert.RepeatInterval } } From 7e63c00bc6e1d01ce63ea42b699fe199cd11d958 Mon Sep 17 00:00:00 2001 From: Jackson Sabey Date: Sat, 31 Aug 2024 15:28:49 -0700 Subject: [PATCH 6/8] send reminder if last reminder is zero --- watchdog/alerting.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/watchdog/alerting.go b/watchdog/alerting.go index c57a0f7b3..72cacb194 100644 --- a/watchdog/alerting.go +++ b/watchdog/alerting.go @@ -38,7 +38,8 @@ func handleAlertsToTrigger(ep *endpoint.Endpoint, result *endpoint.Result, alert if !ep.LastReminderSent.IsZero() { lastReminder = time.Since(ep.LastReminderSent) } - sendReminder := endpointAlert.Triggered && endpointAlert.RepeatInterval > 0 && lastReminder >= endpointAlert.RepeatInterval + sendReminder := endpointAlert.Triggered && endpointAlert.RepeatInterval > 0 && + (lastReminder == 0 || lastReminder >= endpointAlert.RepeatInterval) // If neither initial alert nor reminder needs to be sent, skip to the next alert if !sendInitialAlert && !sendReminder { if debug { From 45bfd479208fc0cff9d9023f910ecffce5d8400a Mon Sep 17 00:00:00 2001 From: Jackson Sabey Date: Sat, 31 Aug 2024 15:56:23 -0700 Subject: [PATCH 7/8] use last reminder based upon alert type --- config/endpoint/endpoint.go | 5 ++++- config/endpoint/external_endpoint.go | 3 +++ watchdog/alerting.go | 12 ++++++------ 3 files changed, 13 insertions(+), 7 deletions(-) diff --git a/config/endpoint/endpoint.go b/config/endpoint/endpoint.go index e73d40758..bfc57872e 100644 --- a/config/endpoint/endpoint.go +++ b/config/endpoint/endpoint.go @@ -123,7 +123,7 @@ type Endpoint struct { NumberOfSuccessesInARow int `yaml:"-"` // LastReminderSent is the time at which the last reminder was sent for this endpoint. - LastReminderSent time.Time `yaml:"-"` + LastReminderSent map[alert.Type]time.Time `yaml:"-"` } // IsEnabled returns whether the endpoint is enabled or not @@ -193,6 +193,9 @@ func (e *Endpoint) ValidateAndSetDefaults() error { if len(e.Headers) == 0 { e.Headers = make(map[string]string) } + if len(e.LastReminderSent) == 0 { + e.LastReminderSent = make(map[alert.Type]time.Time) + } // Automatically add user agent header if there isn't one specified in the endpoint configuration if _, userAgentHeaderExists := e.Headers[UserAgentHeader]; !userAgentHeaderExists { e.Headers[UserAgentHeader] = GatusUserAgent diff --git a/config/endpoint/external_endpoint.go b/config/endpoint/external_endpoint.go index 58f37fedd..8527187cb 100644 --- a/config/endpoint/external_endpoint.go +++ b/config/endpoint/external_endpoint.go @@ -2,6 +2,7 @@ package endpoint import ( "errors" + "time" "github.com/TwiN/gatus/v5/alerting/alert" ) @@ -75,6 +76,8 @@ func (externalEndpoint *ExternalEndpoint) ToEndpoint() *Endpoint { Enabled: externalEndpoint.Enabled, Name: externalEndpoint.Name, Group: externalEndpoint.Group, + Headers: make(map[string]string), + LastReminderSent: make(map[alert.Type]time.Time), Alerts: externalEndpoint.Alerts, NumberOfFailuresInARow: externalEndpoint.NumberOfFailuresInARow, NumberOfSuccessesInARow: externalEndpoint.NumberOfSuccessesInARow, diff --git a/watchdog/alerting.go b/watchdog/alerting.go index 72cacb194..99d2ee036 100644 --- a/watchdog/alerting.go +++ b/watchdog/alerting.go @@ -35,16 +35,16 @@ func handleAlertsToTrigger(ep *endpoint.Endpoint, result *endpoint.Result, alert sendInitialAlert := !endpointAlert.Triggered // Determine if a reminder should be sent var lastReminder time.Duration - if !ep.LastReminderSent.IsZero() { - lastReminder = time.Since(ep.LastReminderSent) + if lr, ok := ep.LastReminderSent[endpointAlert.Type]; ok && !lr.IsZero() { + lastReminder = time.Since(lr) } sendReminder := endpointAlert.Triggered && endpointAlert.RepeatInterval > 0 && (lastReminder == 0 || lastReminder >= endpointAlert.RepeatInterval) // If neither initial alert nor reminder needs to be sent, skip to the next alert if !sendInitialAlert && !sendReminder { if debug { - log.Printf("[watchdog.handleAlertsToTrigger] Alert for endpoint=%s with description='%s' is not due for triggering or reminding (interval: %s last: %s), skipping", - ep.Name, endpointAlert.GetDescription(), endpointAlert.RepeatInterval, lastReminder) + log.Printf("[watchdog.handleAlertsToTrigger] Alert %s for endpoint=%s with description='%s' is not due for triggering (interval: %s last: %s), skipping", + endpointAlert.Type, ep.Name, endpointAlert.GetDescription(), endpointAlert.RepeatInterval, lastReminder) } continue } @@ -70,7 +70,7 @@ func handleAlertsToTrigger(ep *endpoint.Endpoint, result *endpoint.Result, alert if sendInitialAlert { endpointAlert.Triggered = true } - ep.LastReminderSent = time.Now() + ep.LastReminderSent[endpointAlert.Type] = time.Now() if err := store.Get().UpsertTriggeredEndpointAlert(ep, endpointAlert); err != nil { log.Printf("[watchdog.handleAlertsToTrigger] Failed to persist triggered endpoint alert for endpoint with key=%s: %s", ep.Key(), err.Error()) } @@ -113,7 +113,7 @@ func handleAlertsToResolve(ep *endpoint.Endpoint, result *endpoint.Result, alert } else { log.Printf("[watchdog.handleAlertsToResolve] Not sending alert of type=%s despite being RESOLVED, because the provider wasn't configured properly", endpointAlert.Type) } + ep.LastReminderSent[endpointAlert.Type] = time.Now() } ep.NumberOfFailuresInARow = 0 - ep.LastReminderSent = time.Now() } From 8e496904be57f09d8e5b6b9b0d0169e5cfc69d71 Mon Sep 17 00:00:00 2001 From: Jackson Sabey Date: Sat, 31 Aug 2024 16:33:03 -0700 Subject: [PATCH 8/8] only set RepeatInterval if nil --- alerting/alert/alert.go | 2 +- alerting/provider/provider.go | 2 +- watchdog/alerting.go | 4 ++-- watchdog/alerting_test.go | 30 ++++++++++++++++++++---------- 4 files changed, 24 insertions(+), 14 deletions(-) diff --git a/alerting/alert/alert.go b/alerting/alert/alert.go index b35a18188..e885551cd 100644 --- a/alerting/alert/alert.go +++ b/alerting/alert/alert.go @@ -50,7 +50,7 @@ type Alert struct { ResolveKey string `yaml:"-"` // RepeatInterval is the interval between reminders - RepeatInterval time.Duration `yaml:"repeat-interval,omitempty"` + RepeatInterval *time.Duration `yaml:"repeat-interval"` // Triggered is used to determine whether an alert has been triggered. When an alert is resolved, this value // should be set back to false. It is used to prevent the same alert from going out twice. diff --git a/alerting/provider/provider.go b/alerting/provider/provider.go index 230f73cc0..b66ad52b8 100644 --- a/alerting/provider/provider.go +++ b/alerting/provider/provider.go @@ -57,7 +57,7 @@ func ParseWithDefaultAlert(providerDefaultAlert, endpointAlert *alert.Alert) { if endpointAlert.SuccessThreshold == 0 { endpointAlert.SuccessThreshold = providerDefaultAlert.SuccessThreshold } - if endpointAlert.RepeatInterval == 0 { + if endpointAlert.RepeatInterval == nil { endpointAlert.RepeatInterval = providerDefaultAlert.RepeatInterval } } diff --git a/watchdog/alerting.go b/watchdog/alerting.go index 99d2ee036..8a12d497d 100644 --- a/watchdog/alerting.go +++ b/watchdog/alerting.go @@ -38,8 +38,8 @@ func handleAlertsToTrigger(ep *endpoint.Endpoint, result *endpoint.Result, alert if lr, ok := ep.LastReminderSent[endpointAlert.Type]; ok && !lr.IsZero() { lastReminder = time.Since(lr) } - sendReminder := endpointAlert.Triggered && endpointAlert.RepeatInterval > 0 && - (lastReminder == 0 || lastReminder >= endpointAlert.RepeatInterval) + sendReminder := endpointAlert.Triggered && endpointAlert.RepeatInterval != nil && + *endpointAlert.RepeatInterval > 0 && (lastReminder == 0 || lastReminder >= *endpointAlert.RepeatInterval) // If neither initial alert nor reminder needs to be sent, skip to the next alert if !sendInitialAlert && !sendReminder { if debug { diff --git a/watchdog/alerting_test.go b/watchdog/alerting_test.go index 8b0a61491..8fc75eb46 100644 --- a/watchdog/alerting_test.go +++ b/watchdog/alerting_test.go @@ -39,7 +39,8 @@ func TestHandleAlerting(t *testing.T) { } enabled := true ep := &endpoint.Endpoint{ - URL: "https://example.com", + URL: "https://example.com", + LastReminderSent: make(map[alert.Type]time.Time), Alerts: []*alert.Alert{ { Type: alert.TypeCustom, @@ -83,7 +84,8 @@ func TestHandleAlertingWithBadAlertProvider(t *testing.T) { enabled := true ep := &endpoint.Endpoint{ - URL: "http://example.com", + URL: "http://example.com", + LastReminderSent: make(map[alert.Type]time.Time), Alerts: []*alert.Alert{ { Type: alert.TypeCustom, @@ -118,7 +120,8 @@ func TestHandleAlertingWhenTriggeredAlertIsAlmostResolvedButendpointStartFailing } enabled := true ep := &endpoint.Endpoint{ - URL: "https://example.com", + URL: "https://example.com", + LastReminderSent: make(map[alert.Type]time.Time), Alerts: []*alert.Alert{ { Type: alert.TypeCustom, @@ -153,7 +156,8 @@ func TestHandleAlertingWhenTriggeredAlertIsResolvedButSendOnResolvedIsFalse(t *t enabled := true disabled := false ep := &endpoint.Endpoint{ - URL: "https://example.com", + URL: "https://example.com", + LastReminderSent: make(map[alert.Type]time.Time), Alerts: []*alert.Alert{ { Type: alert.TypeCustom, @@ -185,7 +189,8 @@ func TestHandleAlertingWhenTriggeredAlertIsResolvedPagerDuty(t *testing.T) { } enabled := true ep := &endpoint.Endpoint{ - URL: "https://example.com", + URL: "https://example.com", + LastReminderSent: make(map[alert.Type]time.Time), Alerts: []*alert.Alert{ { Type: alert.TypePagerDuty, @@ -221,7 +226,8 @@ func TestHandleAlertingWhenTriggeredAlertIsResolvedPushover(t *testing.T) { } enabled := true ep := &endpoint.Endpoint{ - URL: "https://example.com", + URL: "https://example.com", + LastReminderSent: make(map[alert.Type]time.Time), Alerts: []*alert.Alert{ { Type: alert.TypePushover, @@ -391,7 +397,8 @@ func TestHandleAlertingWithProviderThatReturnsAnError(t *testing.T) { for _, scenario := range scenarios { t.Run(scenario.Name, func(t *testing.T) { ep := &endpoint.Endpoint{ - URL: "https://example.com", + URL: "https://example.com", + LastReminderSent: make(map[alert.Type]time.Time), Alerts: []*alert.Alert{ { Type: scenario.AlertType, @@ -451,7 +458,8 @@ func TestHandleAlertingWithProviderThatOnlyReturnsErrorOnResolve(t *testing.T) { } enabled := true ep := &endpoint.Endpoint{ - URL: "https://example.com", + URL: "https://example.com", + LastReminderSent: make(map[alert.Type]time.Time), Alerts: []*alert.Alert{ { Type: alert.TypeCustom, @@ -502,8 +510,10 @@ func TestHandleAlertingWithRepeatInterval(t *testing.T) { }, } enabled := true + repeatInterval := 1 * time.Second ep := &endpoint.Endpoint{ - URL: "https://example.com", + URL: "https://example.com", + LastReminderSent: make(map[alert.Type]time.Time), Alerts: []*alert.Alert{ { Type: alert.TypeCustom, @@ -512,7 +522,7 @@ func TestHandleAlertingWithRepeatInterval(t *testing.T) { SuccessThreshold: 3, SendOnResolved: &enabled, Triggered: false, - RepeatInterval: 1 * time.Second, + RepeatInterval: &repeatInterval, }, }, }