Merge pull request #60 from MerzMax/fix/prometheus-rr

Add server-to-server communication, multiple instances
MerzMax · Feb 14, 2022 · c975367 · c975367
2 parents 0794777 + 364d035
commit c975367
Show file tree

Hide file tree

Showing 15 changed files with 374 additions and 60 deletions.
diff --git a/.env.example b/.env.example
@@ -1,2 +1,6 @@
-GRAFANA_ADMIN_USER=admin
-GRAFANA_ADMIN_PASSWORD=secret
+GRAFANA_ADMIN_USER=
+GRAFANA_ADMIN_PASSWORD=
+
+ENABLE_DIST=
+DIST_SERVER_PASSWORD=
+DIST_TOPIC=
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -1,12 +1,29 @@
 version: "3"
 
 services:
-  server:
+
+  traefik:
+    image: "traefik:v2.6"
+    command:
+      - "--api.insecure=true"
+      - "--providers.docker=true"
+      - "--providers.docker.exposedbydefault=false"
+      - "--entrypoints.web.address=:80"
+    ports:
+      - "80:80"
+      - "8080:8080"
+    volumes:
+      - "/var/run/docker.sock:/var/run/docker.sock:ro"
+
+  server-one:
+    image: scale-chat-server
     build:
       context: .
       dockerfile: src/server/Dockerfile
-    ports:
-      - "8080:8080"
+    environment:
+      ENABLE_DIST: "true"
+      DIST_SERVER: "redis:6379"
+      DIST_TOPIC: "messages"
     deploy:
       resources:
           limits:
@@ -15,10 +32,47 @@ services:
           reservations:
             cpus: "0.25"
             memory: 256M
+    depends_on:
+      - redis
+    labels:
+      - "traefik.enable=true"
+      - "traefik.http.routers.chat.service=chat"
+      - "traefik.http.routers.chat.rule=PathPrefix(`/`)"
+      - "traefik.http.services.chat.loadbalancer.sticky.cookie=true"
 
+  server-two:
+    image: scale-chat-server
+    build:
+      context: .
+      dockerfile: src/server/Dockerfile
+    environment:
+      ENABLE_DIST: "true"
+      DIST_SERVER: "redis:6379"
+      DIST_TOPIC: "messages"
+    deploy:
+      resources:
+        limits:
+          cpus: "0.25"
+          memory: 512M
+        reservations:
+          cpus: "0.25"
+          memory: 256M
+    depends_on:
+      - redis
+    labels:
+      - "traefik.enable=true"
+      - "traefik.http.routers.chat.service=chat"
+      - "traefik.http.routers.chat.rule=PathPrefix(`/`)"
+      - "traefik.http.services.chat.loadbalancer.sticky.cookie=true"
+
+  redis: 
+    image: redis:6.2.6-alpine
+    ports:
+      - "6379:6379"
 
   prometheus:
     image: prom/prometheus:v2.31.1
+    user: 117:117
     networks:
       - default
     depends_on:
@@ -28,11 +82,13 @@ services:
     volumes:
       - ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro
       - ./prometheus/data:/prometheus
+      - /var/run/docker.sock:/var/run/docker.sock:ro
 
   grafana:
     image: grafana/grafana:8.2.5
     environment:
       PROMETHEUS_HOST: http://prometheus:9090
+      # When deploying with docker stack deploy the .env file will be ignored and the default config will be initialized
       GF_SECURITY_ADMIN_USER: ${GRAFANA_ADMIN_USER}
       GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD}
     ports:
@@ -49,5 +105,3 @@ services:
       - /sys:/sys:ro
       - /var/lib/docker:/var/lib/docker:ro
     privileged: true
-    depends_on:
-      - server
diff --git a/grafana/dashboards/cadvisor-exporter.json b/grafana/dashboards/cadvisor-exporter.json
@@ -23,7 +23,7 @@
   "fiscalYearStartMonth": 0,
   "gnetId": 14282,
   "graphTooltip": 0,
-  "iteration": 1643972821121,
+  "iteration": 1644332858021,
   "links": [],
   "liveNow": false,
   "panels": [
@@ -81,15 +81,15 @@
       "renderer": "flot",
       "seriesOverrides": [],
       "spaceLength": 10,
-      "stack": true,
+      "stack": false,
       "steppedLine": false,
       "targets": [
         {
           "exemplar": true,
-          "expr": "sum(rate(container_cpu_usage_seconds_total{instance=~\"$host\",name=~\".+server.+\",name=~\".+\"}[60s])) *100",
+          "expr": "sum(rate(container_cpu_usage_seconds_total{instance=~\"$host\",name=~\".+(server|redis).+\",name=~\".+\"}[60s])) by (name)",
           "hide": false,
           "interval": "",
-          "legendFormat": "Server",
+          "legendFormat": "{{name}}",
           "refId": "A"
         }
       ],
@@ -124,8 +124,9 @@
       "yaxes": [
         {
           "$$hashKey": "object:606",
-          "format": "percent",
-          "label": null,
+          "decimals": null,
+          "format": "percentunit",
+          "label": "",
           "logBase": 1,
           "max": null,
           "min": null,
@@ -200,15 +201,15 @@
       "renderer": "flot",
       "seriesOverrides": [],
       "spaceLength": 10,
-      "stack": true,
+      "stack": false,
       "steppedLine": false,
       "targets": [
         {
           "exemplar": true,
-          "expr": "sum(container_memory_rss{instance=~\"$host\",name=~\".+server.+\",name=~\".+\"})",
+          "expr": "sum(container_memory_rss{instance=~\"$host\",name=~\".+(server|redis).+\",name=~\".+\"}) by (name)",
           "hide": false,
           "interval": "",
-          "legendFormat": "Server",
+          "legendFormat": "{{ name }}",
           "refId": "A"
         }
       ],
@@ -305,15 +306,15 @@
       "renderer": "flot",
       "seriesOverrides": [],
       "spaceLength": 10,
-      "stack": true,
+      "stack": false,
       "steppedLine": false,
       "targets": [
         {
           "exemplar": true,
-          "expr": "sum(container_memory_cache{instance=~\"$host\",name=~\".+server.+\",name=~\".+\"})",
+          "expr": "sum(container_memory_cache{instance=~\"$host\",name=~\".+(server|redis).+\",name=~\".+\"}) by (name)",
           "hide": false,
           "interval": "",
-          "legendFormat": "Server",
+          "legendFormat": "{{name}}",
           "refId": "A"
         }
       ],
@@ -432,7 +433,7 @@
       "targets": [
         {
           "exemplar": true,
-          "expr": "sum(rate(container_network_receive_bytes_total{instance=~\"$host\",name=~\"$container\",name=~\".+\"}[60s])) by (name)",
+          "expr": "sum(rate(container_network_receive_bytes_total{instance=~\"$host\",name=~\"$container\",name=~\".+(server|redis).+\"}[60s])) by (name)",
           "hide": false,
           "interval": "",
           "legendFormat": "{{name}}",
@@ -527,7 +528,7 @@
       "targets": [
         {
           "exemplar": true,
-          "expr": "sum(rate(container_network_transmit_bytes_total{instance=~\"$host\",name=~\"$container\",name=~\".+\"}[60s])) by (name)",
+          "expr": "sum(rate(container_network_transmit_bytes_total{instance=~\"$host\",name=~\"$container\",name=~\".+(server|redis).+\"}[60s])) by (name)",
           "interval": "",
           "legendFormat": "{{name}}",
           "refId": "A"
@@ -790,12 +791,12 @@
     ]
   },
   "time": {
-    "from": "now-1h",
+    "from": "now-15m",
     "to": "now"
   },
   "timepicker": {},
   "timezone": "",
   "title": "Cadvisor exporter",
   "uid": "pMEd7m0Mz",
-  "version": 1
+  "version": 2
 }
diff --git a/prometheus/prometheus.yml b/prometheus/prometheus.yml
@@ -7,7 +7,8 @@ scrape_configs:
     metrics_path: /metrics
     static_configs:
       - targets:
-        - server:8081
+          - server-one:8081
+          - server-two:8081
 
   - job_name: cadvisor
     static_configs:

diff --git a/src/chat/message.go b/src/chat/message.go
@@ -11,20 +11,22 @@ type Message struct {
 	Text      string    `json:"text"`
 	Sender    string    `json:"sender"`
 	SentAt    time.Time `json:"sent_at"`
+	Room      string    `json:"room"`
 }
 
-func ParseMessage(data []byte) (Message, error) {
-	var message Message
-	err := json.Unmarshal(data, &message)
+// UnmarshalBinary a given byte array to a Message
+func (msg *Message) UnmarshalBinary(data []byte) error {
+	err := json.Unmarshal(data, msg)
 	if err != nil {
 		log.Printf("Cannot parse message: %v", err)
-		return message, err
+		return err
 	}
-	return message, nil
+	return nil
 }
 
-func (message *Message) ToJSON() ([]byte, error) {
-	data, err := json.Marshal(message)
+// MarshalBinary a given Message to a byte array
+func (msg *Message) MarshalBinary() ([]byte, error) {
+	data, err := json.Marshal(msg)
 	if err != nil {
 		log.Printf("Cannot marshal message: %v", err)
 		return data, err

diff --git a/src/client/client.go b/src/client/client.go
@@ -133,7 +133,8 @@ func (client *Client) receiveHandler(ctx context.Context, waitGroup *sync.WaitGr
 
 			receivedAt := time.Now()
 
-			message, err := chat.ParseMessage(*data)
+			var message chat.Message
+			err := message.UnmarshalBinary(*data)
 			if err != nil {
 				continue
 			}
@@ -189,9 +190,10 @@ func (client *Client) sendHandler(ctx context.Context, waitGroup *sync.WaitGroup
 				Text:      text,
 				Sender:    client.id,
 				SentAt:    time.Now(),
+				Room:      client.Room,
 			}
 
-			data, err := message.ToJSON()
+			data, err := message.MarshalBinary()
 			if err != nil {
 				continue
 			}

diff --git a/src/go.mod b/src/go.mod
@@ -4,21 +4,24 @@ go 1.17
 
 require (
 	github.com/go-echarts/go-echarts/v2 v2.2.4
+	github.com/go-redis/redis/v8 v8.11.4
 	github.com/google/uuid v1.3.0
 	github.com/gorilla/mux v1.8.0
 	github.com/gorilla/websocket v1.4.2
+	github.com/joho/godotenv v1.4.0
 	github.com/montanaflynn/stats v0.6.6
 	github.com/prometheus/client_golang v1.11.0
 )
 
 require (
 	github.com/beorn7/perks v1.0.1 // indirect
-	github.com/cespare/xxhash/v2 v2.1.1 // indirect
-	github.com/golang/protobuf v1.4.3 // indirect
+	github.com/cespare/xxhash/v2 v2.1.2 // indirect
+	github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
+	github.com/golang/protobuf v1.5.2 // indirect
 	github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
 	github.com/prometheus/client_model v0.2.0 // indirect
 	github.com/prometheus/common v0.26.0 // indirect
 	github.com/prometheus/procfs v0.6.0 // indirect
 	golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40 // indirect
-	google.golang.org/protobuf v1.26.0-rc.1 // indirect
+	google.golang.org/protobuf v1.26.0 // indirect
 )