feat: DEVOPS-1632 opentelemetry local setup

docker-compose.yaml

@@ -6,7 +6,7 @@ services:
       - node3 
     environment:
       RUST_BACKTRACE: 1
-      RUST_LOG: zilliqa=info
+      RUST_LOG: zilliqa=info,opentelemetry=trace
     image: zq2-node0
     build:
       context: .
@@ -27,7 +27,7 @@ services:
       - node3
     environment:
       RUST_BACKTRACE: 1
-      RUST_LOG: zilliqa=info
+      RUST_LOG: zilliqa=info,opentelemetry=trace
     container_name: zilliqanode1
     image: zq2-node0
     volumes:
@@ -43,7 +43,7 @@ services:
       - node3
     environment:
       RUST_BACKTRACE: 1
-      RUST_LOG: zilliqa=info
+      RUST_LOG: zilliqa=info,opentelemetry=trace
     container_name: zilliqanode2
     image: zq2-node0
     volumes:
@@ -57,7 +57,7 @@ services:
   node3:
     environment:
       RUST_BACKTRACE: 1
-      RUST_LOG: zilliqa=info
+      RUST_LOG: zilliqa=info,opentelemetry=trace
     container_name: zilliqanode3
     image: zq2-node0
     volumes:
@@ -99,6 +99,40 @@ services:
       testnet:
         ipv4_address: 198.51.100.10
 
+  otel-collector:
+    image: otel/opentelemetry-collector-contrib:latest
+    command: ["--config=/etc/otel-collector-config.yaml"]
+    volumes:
+      - ./infra/otel-collector-config.yaml:/etc/otel-collector-config.yaml
+    ports:
+      - "4317:4317"
+      - "9011:8888"
+    networks:
+      - testnet
+
+  mimir:
+    image: grafana/mimir:latest
+    volumes:
+      - "./infra/mimir-config.yaml:/etc/mimir/config.yaml"
+    command:
+      - "./mimir"
+      - "--config.file"
+      - "/etc/mimir/config.yaml"
+    ports:
+      - "9009:9009"
+    networks:
+      - testnet
+
+  grafana:
+    image: grafana/grafana-oss:latest
+    environment:
+      - GF_AUTH_ANONYMOUS_ENABLED=true
+      - GF_AUTH_ANONYMOUS_ORG_ROLE=Admin
+    ports:
+      - "9010:3000"
+    networks:
+      - testnet
+
 networks:
   testnet:
     driver: bridge

infra/config_docker.toml

@@ -4,6 +4,9 @@ bootstrap_address = [
 ]
 p2p_port = 5643
 
+# OpenTelemetry parameters
+otlp_collector_endpoint = "http://otel-collector:4317"
+
 [[nodes]]
 api_servers = [
     { port = 4201, enabled_apis = [ "admin", "erigon", "eth", "net", "ots", "trace", "txpool", "web3", "zilliqa" ] },
@@ -77,13 +80,14 @@ consensus.genesis_accounts = [
     ],
 ]
 
-# speed up local/docker epochs
+# Speed up local/docker epochs
 consensus.blocks_per_epoch = 36
 
 # Reward parameters
 consensus.rewards_per_hour = "51_000_000_000_000_000_000_000"
 consensus.blocks_per_hour = 3600
 consensus.minimum_stake = "10_000_000_000_000_000_000_000_000"
+
 # Gas parameters
 consensus.eth_block_gas_limit = 84000000
-consensus.gas_price = "4_761_904_800_000"
+consensus.gas_price = "4_761_904_800_000"

infra/mimir.yaml → infra/mimir-config.yaml

@@ -3,6 +3,12 @@ usage_stats:
 
 multitenancy_enabled: false
 
+memberlist:
+  bind_port: 7946
+  advertise_addr: 0.0.0.0
+  join_members:
+    - 0.0.0.0:7946
+
 blocks_storage:
   backend: filesystem
   bucket_store:
@@ -20,13 +26,13 @@ compactor:
 
 distributor:
   ring:
-    instance_addr: 127.0.0.1
+    instance_addr: 0.0.0.0
     kvstore:
       store: memberlist
 
 ingester:
   ring:
-    instance_addr: 127.0.0.1
+    instance_addr: 0.0.0.0
     kvstore:
       store: memberlist
     replication_factor: 1

z2/resources/config.tera.toml

@@ -4,6 +4,8 @@ p2p_port = 3333
 bootstrap_address = [ "{{ bootstrap_peer_id }}", "/dns/{{ bootstrap_endpoint }}/tcp/3333" ]
 {%- endif %}
 
+otlp_collector_endpoint = "http://localhost:4317"
+
 [[nodes]]
 eth_chain_id = {{ eth_chain_id }}
 allowed_timestamp_skew = { secs = 60, nanos = 0 }

z2/resources/node_provision.tera.py

@@ -14,7 +14,6 @@
 templatefile() vars:
 - checkpoint_url, the ZQ2 checkpoint URL used for recover the validator nodes
 - persistence_url, the ZQ2 persistence URL used for recover the network
-- conifg, the ZQ2 validators configuration file
 - docker_image, the ZQ2 docker image (incl. version)
 - secret_key, the ZQ2 node secret key
 - role, the node role: validator or apps
@@ -121,7 +120,7 @@ def health():
 """
 
 ZQ2_SCRIPT="""#!/bin/bash
-echo yes |  gcloud auth configure-docker asia-docker.pkg.dev,europe-docker.pkg.dev
+echo yes | gcloud auth configure-docker asia-docker.pkg.dev,europe-docker.pkg.dev
 
 ZQ2_IMAGE="{{ docker_image }}"
 
@@ -130,7 +129,7 @@ def health():
     docker container prune -f
     docker run -td -p 3333:3333/udp -p 4201:4201 -p 4202:4202 --net=host --name zilliqa-""" + VERSIONS.get('zilliqa') + """ \
     --log-driver json-file --log-opt max-size=1g --log-opt max-file=30 \
-    -e RUST_LOG="zilliqa=trace" -e RUST_BACKTRACE=1 \
+    -e RUST_LOG="zilliqa=trace,opentelemetry=trace" -e RUST_BACKTRACE=1 \
     --restart=unless-stopped \
     -v /config.toml:/config.toml -v /zilliqa.log:/zilliqa.log -v /data:/data \
     """ + mount_checkpoint_file() + """ ${ZQ2_IMAGE} ${1} --log-json
@@ -256,11 +255,14 @@ def health():
 
 OPS_AGENT_INSTALL_SCRIPT_URL="https://dl.google.com/cloudagents/add-google-cloud-ops-agent-repo.sh"
 
-
 OPS_AGENT_CONFIG_FILE="/etc/google-cloud-ops-agent/config.yaml"
 
-
 OPS_AGENT_CONFIG="""
+combined:
+  receivers:
+    otlp:
+      type: otlp
+      metrics_mode: googlecloudmonitoring
 logging:
   receivers:
     zilliqa:
@@ -310,6 +312,13 @@ def health():
       default_pipeline:
         receivers: [hostmetrics]
         processors: [metrics_filter]
+      otlp:
+        receivers: [otlp]
+traces:
+  service:
+    pipelines:
+      otlp:
+        receivers: [otlp]
 """
 
 LOGROTATE_CONFIG="""

z2/resources/otel-compose.yaml

@@ -3,7 +3,7 @@ services:
     image: otel/opentelemetry-collector:latest
     command: ["--config=/etc/otel-collector-config.yaml"]
     volumes:
-      - ./otel-config.yaml:/etc/otel-collector-config.yaml
+      - ./otel-collector-config.yaml:/etc/otel-collector-config.yaml
     ports:
       - "4317:4317"
       - "9011:8888"
@@ -25,4 +25,4 @@ services:
       - GF_AUTH_ANONYMOUS_ENABLED=true
       - GF_AUTH_ANONYMOUS_ORG_ROLE=Admin
     ports:
-      - "9010:3000"
+      - "9010:3000"