Configurable error handling and retrying in HTTP target

assets/docs/configuration/targets/default.hcl

@@ -0,0 +1,24 @@
+target {
+  use "http" {
+
+    // by default no custom invalid/retryable rules so we use current default retrying strategy for target failures in `sourceWriteFunc` (in cli.go) 
+    response_handler {
+      success = [{ http_status: ["2**"]}] 
+      invalid/badrow = []
+      retryable = [] 
+
+      retry_strategies {
+        transient {
+          policy: "exponential"
+          max_attempts: 5
+          delay: "2 seconds"
+        }
+        setup {
+          policy: "exponential" 
+          max_attempts: 10000 //very high value making it basically unlimited
+          delay: "30 seconds"
+        }
+      }
+    }
+  }
+}

assets/docs/configuration/targets/http_response_handler.hcl

@@ -0,0 +1,60 @@
+target {
+  use "http" {
+
+    response_handler {
+
+     // everything is fine, we can ack our data from source.
+     // Can such configuration be useful or it's overkill and we should commit to 2** all the time?
+      success = [
+        //we can have concrete status or wildcard using *
+        { http_status: ["2**"]}
+      ] 
+
+      // data is for some reason not accepted by target, there is no point of retrying. Just send it to bad/failed target and unblock processing.
+      invalid/badrow = [
+
+        //first rule...
+        { http_status: ["41*"]},
+
+        //second rule...
+        { 
+          http_status: ["499", "424"],
+
+          //not only http status but check if response body matches. Extract some part of body (e.g. error code) and check if it's equal/not equal to value.
+          body {
+            path: "some jq here??"
+            equal/not_equal: "some value"
+          }
+        }
+      ]
+
+      // For these retryable errors (assuming we have health check):
+      // - set app status as unhealthy for each failed retry
+      // - set app status as healthy after successful retry
+      // - if we run out of attempts =>  exit/crash/restart
+      // - if we don't have max attempts (e.g. like setup errors), we don't get stuck retrying indefinitely, because we are protected by health check. If app is unhealthy for extended period time, it will be killed eventually by kubernetes. 
+      retryable = [
+        { http_status: ["503"], strategy: "transient"}, 
+        { 
+          http_status: ["403"],
+          strategy: "setup",
+          alert: "This is alert sent to ops1. You can configure message for specific error code"
+        }
+      ]
+
+       // and the list of retry strategies, you can define how many you want and then reference them in your 'retryable' rules
+       retry_strategies {
+          transient { 
+            policy: "exponential"
+            max_attempts: 5
+            delay: "2 seconds"
+          }
+          setup {
+            policy: "exponential" 
+            max_attempts: 10000
+            delay: "30 seconds"
+          }
+      }
+    }
+  }
+}

pkg/health/health_check.go

@@ -0,0 +1,30 @@
+/**
+ * Copyright (c) 2020-present Snowplow Analytics Ltd.
+ * All rights reserved.
+ *
+ * This software is made available by Snowplow Analytics, Ltd.,
+ * under the terms of the Snowplow Limited Use License Agreement, Version 1.0
+ * located at https://docs.snowplow.io/limited-use-license-1.0
+ * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION
+ * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT.
+ */
+
+package health 
+
+import (
+  "sync/atomic"
+)
+
+var isHealthy atomic.Bool
+
+func SetHealthy() {
+  isHealthy.Store(true)
+}
+
+func SetUnhealthy() {
+  isHealthy.Store(false)
+}
+
+func IsHealthy() bool {
+  return isHealthy.Load()
+}

pkg/monitoring/monitoring.go

@@ -0,0 +1,45 @@
+/**
+ * Copyright (c) 2020-present Snowplow Analytics Ltd.
+ * All rights reserved.
+ *
+ * This software is made available by Snowplow Analytics, Ltd.,
+ * under the terms of the Snowplow Limited Use License Agreement, Version 1.0
+ * located at https://docs.snowplow.io/limited-use-license-1.0
+ * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION
+ * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT.
+ */
+
+package monitoring
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"net"
+	"net/http"
+	"net/url"
+	"time"
+
+	"github.com/hashicorp/go-multierror"
+	"github.com/pkg/errors"
+	log "github.com/sirupsen/logrus"
+
+	"github.com/snowplow/snowbridge/pkg/common"
+	"github.com/snowplow/snowbridge/pkg/models"
+
+	"golang.org/x/oauth2"
+)
+
+type Alert struct {
+  Message string
+}
+
+type Monitoring struct {
+	client            *http.Client
+	httpURL           string
+}
+
+func (m *Monitoring) SendAlert (alert Alert) {
+   m.client.Do(....)
+}