From aea41a27ef65011261a6c6d10d98c72729fbbe9f Mon Sep 17 00:00:00 2001 From: kvermeul Date: Wed, 6 Sep 2023 08:30:32 +0000 Subject: [PATCH] adding monitoring --- .dockerignore | 2 ++ docker-compose.yaml | 31 +++++++++++++++++++++---------- go.mod | 1 + go.sum | 7 +++++++ main.go | 32 +++++++++++++++++++++++++++++++- 5 files changed, 62 insertions(+), 11 deletions(-) create mode 100644 .dockerignore diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..efab184 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,2 @@ +local/ +docker-compose-old.yaml \ No newline at end of file diff --git a/docker-compose.yaml b/docker-compose.yaml index f0444fa..2a5c4b3 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -7,17 +7,21 @@ services: - ./local:/local # NOTE: All service containers will use the same network and IP. All ports # must be configured on the first. - ports: - - target: 9990 - published: 9990 - protocol: tcp - mode: bridge - - target: 9991 - published: 9991 - protocol: tcp - mode: bridge + + # If we are using network mode host, we cannot publish ports. + # Do not know in which env the MLab sidecar is deployed, so commenting for now. + + # ports: + # - target: 2222 + # published: 2222 + # protocol: tcp + # mode: bridge + # - target: 9991 + # published: 9991 + # protocol: tcp + # mode: bridge command: - - -prometheusx.listen-address=:9990 + - -prometheusx.listen-address=:2222 - -output=/local/tcpinfo - -tcpinfo.eventsocket=/local/tcpevents.sock - -anonymize.ip=none @@ -36,10 +40,17 @@ services: restart_policy: condition: always delay: 5s + # ports: + # - target: 2112 + # published: 2112 + # protocol: tcp + # mode: bridge command: - -tcpinfo.eventsocket=/local/tcpevents.sock - -revtr.hostname=revtr.ccs.neu.edu - -revtr.grpcPort=9999 - -revtr.APIKey=7vg3XfyGIJZL92Ql - -revtr.sampling=5 + - -prometheus.port=2112 - -loglevel=debug + diff --git a/go.mod b/go.mod index f4c5027..5787a75 100644 --- a/go.mod +++ b/go.mod @@ -8,6 +8,7 @@ require ( github.com/grpc-ecosystem/grpc-gateway v1.16.0 github.com/m-lab/go v1.4.1 github.com/m-lab/tcp-info v1.5.3 + github.com/prometheus/client_golang v1.7.1 // indirect github.com/sirupsen/logrus v1.4.2 google.golang.org/grpc v1.33.1 google.golang.org/protobuf v1.23.0 diff --git a/go.sum b/go.sum index bd57110..50a5fd4 100644 --- a/go.sum +++ b/go.sum @@ -37,8 +37,10 @@ github.com/araddon/dateparse v0.0.0-20200409225146-d820a6159ab1 h1:TEBmxO80TM04L github.com/araddon/dateparse v0.0.0-20200409225146-d820a6159ab1/go.mod h1:SLqhdZcd+dF3TEVL2RMoob5bBP5R1P1qkox+HtCBgGI= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= @@ -136,6 +138,7 @@ github.com/m-lab/go v1.4.1/go.mod h1:woT26L9Hf07juZGHe7Z4WveV7MM6NS6vQaaWzRQnab4 github.com/m-lab/tcp-info v1.5.3 h1:4IspTPcNc8D8LNRvuFnID8gDiz+hxPAtYvpKZaiGGe8= github.com/m-lab/tcp-info v1.5.3/go.mod h1:bkvI4qbjB6QVC2tsLSHqf5OnIYcmuLEVjo7+8YA56Kg= github.com/m-lab/uuid-annotator v0.4.1/go.mod h1:f/zvgcc5A3HQ1Y63HWpbBVXNcsJwQ4uRIOqsF/nyto8= +github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -148,15 +151,19 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= +github.com/prometheus/client_golang v1.7.1 h1:NTGy1Ja9pByO+xAeH/qiWnLrKtr3hJPNjaVUwnjpdpA= github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/common v0.10.0 h1:RyRA7RzGXQZiW+tGMr7sxa85G1z0yOpM1qq5c8lNawc= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/procfs v0.1.3 h1:F0+tqvhOksq22sc6iCHF5WGlWjdwj92p0udFh1VFBS8= github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= diff --git a/main.go b/main.go index e82da4f..91ae416 100644 --- a/main.go +++ b/main.go @@ -22,6 +22,11 @@ import ( "github.com/m-lab/tcp-info/inetdiag" "google.golang.org/grpc" "google.golang.org/grpc/credentials" + + // Monitoring + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" + "github.com/prometheus/client_golang/prometheus/promhttp" ) var ( @@ -34,11 +39,27 @@ var ( revtrAPIKey = flag.String("revtr.APIKey", "", "The API key used by the M-Lab nodes to call the revtr API") // revtrHostname is the hostname of the server hosting the Revtr API revtrHostname = flag.String("revtr.hostname", "", "The hostname of the revtr API server") - revtrSampling = flag.Int("revtr.sampling", 0, "Only run 1 over revtr.sample revtrs to not overload the system") + revtrSampling = flag.Int("revtr.sampling", 0, "Only run 1 / revtr.sample revtrs to not overload the system") + prometheusPort = flag.Int("prometheus.port", 2112, "Prometheus port to run on") + revtrTestSrc = "129.10.113.200" revtrTestSite = "fring2" + ) + +var ( + + revtrAPICallsMonitor = promauto.NewCounter(prometheus.CounterOpts{ + Name: "Reverse Traceroute API calls", + Help: "Reverse Traceroute API calls to the Revtr system", +}) + + revtrSampleMonitor = promauto.NewCounter(prometheus.CounterOpts{ + Name: "Reverse Traceroute measurements sent", + Help: "Reverse Traceroute measurements sent to the Revtr system", + }) +) // event contains fields for an open event. type event struct { timestamp time.Time @@ -82,12 +103,14 @@ func callRevtr(client *revtrpb.RevtrClient, revtrMeasurements []*revtrpb.RevtrMe for i, revtrMeasurement := range(revtrMeasurements) { if i % revtrSampling == 0 { revtrMeasurementsSampled = append(revtrMeasurementsSampled, revtrMeasurement) + revtrSampleMonitor.Inc() } } logger.Debugf("Sending %d reverse traceroutes to the revtr server because of sampling 1 on %d", len(revtrMeasurementsSampled), revtrSampling) + revtrAPICallsMonitor.Inc() _, err := (*client).RunRevtr(ctx, &revtrpb.RunRevtrReq{ Revtrs : revtrMeasurementsSampled, Auth: revtrAPIKey, @@ -345,5 +368,12 @@ func main() { // the given handler. go eventsocket.MustRun(mainCtx, *eventsocket.Filename, h) + http.Handle("/metrics", promhttp.Handler()) + go func() { + http.ListenAndServe(":" + strconv.FormatInt(int64(*prometheusPort), 10), nil) + }() + + + <-mainCtx.Done() }