adding monitoring

.dockerignore

@@ -0,0 +1,2 @@
+local/
+docker-compose-old.yaml

docker-compose.yaml

@@ -7,17 +7,21 @@ services:
       - ./local:/local
     # NOTE: All service containers will use the same network and IP. All ports
     # must be configured on the first.
-    ports:
-      - target: 9990
-        published: 9990
-        protocol: tcp
-        mode: bridge
-      - target: 9991
-        published: 9991
-        protocol: tcp
-        mode: bridge
+
+    # If we are using network mode host, we cannot publish ports. 
+    # Do not know in which env the MLab sidecar is deployed, so commenting for now. 
+
+    # ports:
+    #   - target: 2222
+    #     published: 2222
+    #     protocol: tcp
+    #     mode: bridge
+    #   - target: 9991
+    #     published: 9991
+    #     protocol: tcp
+    #     mode: bridge
     command:
-      - -prometheusx.listen-address=:9990
+      - -prometheusx.listen-address=:2222
       - -output=/local/tcpinfo
       - -tcpinfo.eventsocket=/local/tcpevents.sock
       - -anonymize.ip=none
@@ -36,10 +40,17 @@ services:
       restart_policy:
         condition: always
         delay: 5s
+    # ports:
+    #   - target: 2112
+    #     published: 2112
+    #     protocol: tcp
+    #     mode: bridge
     command:
       - -tcpinfo.eventsocket=/local/tcpevents.sock 
       - -revtr.hostname=revtr.ccs.neu.edu
       - -revtr.grpcPort=9999
       - -revtr.APIKey=7vg3XfyGIJZL92Ql
       - -revtr.sampling=5
+      - -prometheus.port=2112
       - -loglevel=debug
+

go.mod

@@ -8,6 +8,7 @@ require (
 	github.com/grpc-ecosystem/grpc-gateway v1.16.0
 	github.com/m-lab/go v1.4.1
 	github.com/m-lab/tcp-info v1.5.3
+	github.com/prometheus/client_golang v1.7.1 // indirect
 	github.com/sirupsen/logrus v1.4.2
 	google.golang.org/grpc v1.33.1
 	google.golang.org/protobuf v1.23.0

go.sum

@@ -37,8 +37,10 @@ github.com/araddon/dateparse v0.0.0-20200409225146-d820a6159ab1 h1:TEBmxO80TM04L
 github.com/araddon/dateparse v0.0.0-20200409225146-d820a6159ab1/go.mod h1:SLqhdZcd+dF3TEVL2RMoob5bBP5R1P1qkox+HtCBgGI=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
+github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
+github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY=
 github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
 github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
@@ -136,6 +138,7 @@ github.com/m-lab/go v1.4.1/go.mod h1:woT26L9Hf07juZGHe7Z4WveV7MM6NS6vQaaWzRQnab4
 github.com/m-lab/tcp-info v1.5.3 h1:4IspTPcNc8D8LNRvuFnID8gDiz+hxPAtYvpKZaiGGe8=
 github.com/m-lab/tcp-info v1.5.3/go.mod h1:bkvI4qbjB6QVC2tsLSHqf5OnIYcmuLEVjo7+8YA56Kg=
 github.com/m-lab/uuid-annotator v0.4.1/go.mod h1:f/zvgcc5A3HQ1Y63HWpbBVXNcsJwQ4uRIOqsF/nyto8=
+github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
@@ -148,15 +151,19 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
 github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
+github.com/prometheus/client_golang v1.7.1 h1:NTGy1Ja9pByO+xAeH/qiWnLrKtr3hJPNjaVUwnjpdpA=
 github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M=
 github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
 github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M=
 github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
+github.com/prometheus/common v0.10.0 h1:RyRA7RzGXQZiW+tGMr7sxa85G1z0yOpM1qq5c8lNawc=
 github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo=
 github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
 github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
+github.com/prometheus/procfs v0.1.3 h1:F0+tqvhOksq22sc6iCHF5WGlWjdwj92p0udFh1VFBS8=
 github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
 github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=

main.go

@@ -22,6 +22,11 @@ import (
 	"github.com/m-lab/tcp-info/inetdiag"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/credentials"
+
+	// Monitoring
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
 )
 
 var (
@@ -34,11 +39,27 @@ var (
 	revtrAPIKey = flag.String("revtr.APIKey", "", "The API key used by the M-Lab nodes to call the revtr API")
 	// revtrHostname is the hostname of the server hosting the Revtr API
 	revtrHostname = flag.String("revtr.hostname", "", "The hostname of the revtr API server")
-	revtrSampling = flag.Int("revtr.sampling", 0, "Only run 1 over revtr.sample revtrs to not overload the system")
+	revtrSampling = flag.Int("revtr.sampling", 0, "Only run 1 / revtr.sample revtrs to not overload the system")
+	prometheusPort = flag.Int("prometheus.port", 2112, "Prometheus port to run on")
+
 	revtrTestSrc = "129.10.113.200"
 	revtrTestSite = "fring2"
+
 )
 
+
+var (
+
+	revtrAPICallsMonitor = promauto.NewCounter(prometheus.CounterOpts{
+		Name: "Reverse Traceroute API calls",
+		Help: "Reverse Traceroute API calls to the Revtr system",
+})
+
+	revtrSampleMonitor = promauto.NewCounter(prometheus.CounterOpts{
+			Name: "Reverse Traceroute measurements sent",
+			Help: "Reverse Traceroute measurements sent to the Revtr system",
+	})
+)
 // event contains fields for an open event.
 type event struct {
 	timestamp time.Time
@@ -82,12 +103,14 @@ func callRevtr(client *revtrpb.RevtrClient, revtrMeasurements []*revtrpb.RevtrMe
 	for i, revtrMeasurement := range(revtrMeasurements) {
 		if i % revtrSampling == 0 {
 			revtrMeasurementsSampled = append(revtrMeasurementsSampled, revtrMeasurement)
+			revtrSampleMonitor.Inc()
 		}
 	}
 
 	logger.Debugf("Sending %d reverse traceroutes to the revtr server because of sampling 1 on %d", 
 	len(revtrMeasurementsSampled), revtrSampling)
 
+	revtrAPICallsMonitor.Inc()
 	_, err := (*client).RunRevtr(ctx, &revtrpb.RunRevtrReq{
 		Revtrs : revtrMeasurementsSampled,
 		Auth: revtrAPIKey,
@@ -345,5 +368,12 @@ func main() {
 	// the given handler.
 	go eventsocket.MustRun(mainCtx, *eventsocket.Filename, h)
 
+	http.Handle("/metrics", promhttp.Handler())
+	go func() {
+		http.ListenAndServe(":" + strconv.FormatInt(int64(*prometheusPort), 10), nil)
+	}()
+
+
+
 	<-mainCtx.Done()
 }