Add compression level to plot metrics. Gather plot metrics from all h…

…arvesters via the farmer as an alternative to running this on every harvester

go.mod

@@ -3,7 +3,7 @@ module github.com/chia-network/chia-exporter
 go 1.18
 
 require (
-	github.com/chia-network/go-chia-libs v0.4.0
+	github.com/chia-network/go-chia-libs v0.4.1-0.20230821182757-8e649e503882
 	github.com/chia-network/go-modules v0.0.4
 	github.com/oschwald/maxminddb-golang v1.12.0
 	github.com/prometheus/client_golang v1.16.0

go.sum

@@ -45,6 +45,8 @@ github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj
 github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/chia-network/go-chia-libs v0.4.0 h1:KmBIJAtniqaFqyB+IKLPxEEKEtx89kBgP3tC4lgIT80=
 github.com/chia-network/go-chia-libs v0.4.0/go.mod h1:cLTizmlrAoyfL+PGLS5G7MT+Q288wtUHCneIDrIp7Mc=
+github.com/chia-network/go-chia-libs v0.4.1-0.20230821182757-8e649e503882 h1:4aP3hjM1YgWMLDqLql0t9vRdL3IVVxLGwJ7D799EpEg=
+github.com/chia-network/go-chia-libs v0.4.1-0.20230821182757-8e649e503882/go.mod h1:cLTizmlrAoyfL+PGLS5G7MT+Q288wtUHCneIDrIp7Mc=
 github.com/chia-network/go-modules v0.0.4 h1:XlCcuT4j1krLvsFT1Y49Un5xORwcTc8jjE4SHih7OTI=
 github.com/chia-network/go-modules v0.0.4/go.mod h1:JP8mG/9ieE76VcGIbzD5G3/4YDmvNhRryiQwp8GQr1U=
 github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=

internal/metrics/farmer.go

@@ -2,6 +2,7 @@ package metrics
 
 import (
 	"encoding/json"
+	"fmt"
 	"time"
 
 	"github.com/chia-network/go-chia-libs/pkg/rpc"
@@ -31,6 +32,10 @@ type FarmerServiceMetrics struct {
 
 	// Proof Metrics
 	proofsFound *wrappedPrometheus.LazyCounter
+
+	// Remote Harvester Plot Counts
+	plotFilesize *prometheus.GaugeVec
+	plotCount    *prometheus.GaugeVec
 }
 
 // InitMetrics sets all the metrics properties
@@ -46,6 +51,11 @@ func (s *FarmerServiceMetrics) InitMetrics() {
 
 	// Proof Metrics
 	s.proofsFound = s.metrics.newCounter(chiaServiceFarmer, "proofs_found", "Number of proofs found since the exporter has been running")
+
+	// Remote harvester plot counts
+	plotLabels := []string{"host", "size", "type", "compression"}
+	s.plotFilesize = s.metrics.newGaugeVec(chiaServiceFarmer, "plot_filesize", "Filesize of plots separated by harvester", plotLabels)
+	s.plotCount = s.metrics.newGaugeVec(chiaServiceFarmer, "plot_count", "Number of plots separated by harvester", plotLabels)
 }
 
 // InitialData is called on startup of the metrics server, to allow seeding metrics with current/initial data
@@ -86,6 +96,31 @@ func (s *FarmerServiceMetrics) ReceiveResponse(resp *types.WebsocketResponse) {
 // GetConnections handler for get_connections events
 func (s *FarmerServiceMetrics) GetConnections(resp *types.WebsocketResponse) {
 	connectionCountHelper(resp, s.connectionCount)
+	harvesters, _, err := s.metrics.httpClient.FarmerService.GetHarvesters(&rpc.FarmerGetHarvestersOptions{})
+	if err != nil {
+		log.Errorf("farmer: Error getting harvesters: %s\n", err.Error())
+		return
+	}
+
+	for _, harvester := range harvesters.Harvesters {
+		plotSize, plotCount := PlotSizeCountHelper(harvester.Plots)
+
+		// Now we can set the gauges with the calculated total values
+		// Labels: "host", "size", "type", "compression"
+		for kSize, cLevels := range plotSize {
+			for cLevel, fileSizes := range cLevels {
+				s.plotFilesize.WithLabelValues(harvester.Connection.Host, fmt.Sprintf("%d", kSize), "og", fmt.Sprintf("%d", cLevel)).Set(float64(fileSizes[PlotTypeOg]))
+				s.plotFilesize.WithLabelValues(harvester.Connection.Host, fmt.Sprintf("%d", kSize), "pool", fmt.Sprintf("%d", cLevel)).Set(float64(fileSizes[PlotTypePool]))
+			}
+		}
+
+		for kSize, cLevelsByType := range plotCount {
+			for cLevel, plotCountByType := range cLevelsByType {
+				s.plotCount.WithLabelValues(harvester.Connection.Host, fmt.Sprintf("%d", kSize), "og", fmt.Sprintf("%d", cLevel)).Set(float64(plotCountByType[PlotTypeOg]))
+				s.plotCount.WithLabelValues(harvester.Connection.Host, fmt.Sprintf("%d", kSize), "pool", fmt.Sprintf("%d", cLevel)).Set(float64(plotCountByType[PlotTypePool]))
+			}
+		}
+	}
 }
 
 // SubmittedPartial handles a received submitted_partial event

internal/metrics/harvester.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"time"
 
+	"github.com/chia-network/go-chia-libs/pkg/protocols"
 	"github.com/chia-network/go-chia-libs/pkg/rpc"
 	"github.com/chia-network/go-chia-libs/pkg/types"
 	"github.com/prometheus/client_golang/prometheus"
@@ -44,9 +45,10 @@ func (s *HarvesterServiceMetrics) InitMetrics() {
 	// Connection Metrics
 	s.connectionCount = s.metrics.newGaugeVec(chiaServiceHarvester, "connection_count", "Number of active connections for each type of peer", []string{"node_type"})
 
+	plotLabels := []string{"size", "type", "compression"}
 	s.totalPlots = s.metrics.newGauge(chiaServiceHarvester, "total_plots", "Total number of plots on this harvester")
-	s.plotFilesize = s.metrics.newGaugeVec(chiaServiceHarvester, "plot_filesize", "Total filesize of plots on this harvester, by K size", []string{"size", "type"})
-	s.plotCount = s.metrics.newGaugeVec(chiaServiceHarvester, "plot_count", "Total count of plots on this harvester, by K size", []string{"size", "type"})
+	s.plotFilesize = s.metrics.newGaugeVec(chiaServiceHarvester, "plot_filesize", "Total filesize of plots on this harvester, by K size", plotLabels)
+	s.plotCount = s.metrics.newGaugeVec(chiaServiceHarvester, "plot_count", "Total count of plots on this harvester, by K size", plotLabels)
 
 	s.totalFoundProofs = s.metrics.newCounter(chiaServiceHarvester, "total_found_proofs", "Counter of total found proofs since the exporter started")
 	s.lastFoundProofs = s.metrics.newGauge(chiaServiceHarvester, "last_found_proofs", "Number of proofs found for the last farmer_info event")
@@ -155,55 +157,79 @@ func (s *HarvesterServiceMetrics) GetPlots(resp *types.WebsocketResponse) {
 	s.ProcessGetPlots(plots)
 }
 
+// PlotType is the type of plot (og or pool)
+type PlotType uint8
+
+const (
+	// PlotTypeOg is the original plot format, no plotNFT
+	PlotTypeOg = PlotType(0)
+	// PlotTypePool is the new plotNFT plot format
+	PlotTypePool = PlotType(1)
+)
+
 // ProcessGetPlots processes the `GetPlotsResponse` from get_plots so that we can use this with websockets or HTTP RPC requests
 func (s *HarvesterServiceMetrics) ProcessGetPlots(plots *rpc.HarvesterGetPlotsResponse) {
-	// First, iterate through all the plots to get totals for each ksize
-	type plotType uint8
-	plotTypeOg := plotType(0)
-	plotTypePool := plotType(1)
+	plotSize, plotCount := PlotSizeCountHelper(plots.Plots.OrEmpty())
 
-	plotSize := map[uint8]map[plotType]uint64{}
-	plotCount := map[uint8]map[plotType]uint64{}
+	// Now we can set the gauges with the calculated total values
+	// Labels: "size", "type", "compression"
+	for kSize, cLevels := range plotSize {
+		for cLevel, fileSizes := range cLevels {
+			s.plotFilesize.WithLabelValues(fmt.Sprintf("%d", kSize), "og", fmt.Sprintf("%d", cLevel)).Set(float64(fileSizes[PlotTypeOg]))
+			s.plotFilesize.WithLabelValues(fmt.Sprintf("%d", kSize), "pool", fmt.Sprintf("%d", cLevel)).Set(float64(fileSizes[PlotTypePool]))
+		}
+	}
 
-	for _, plot := range plots.Plots.OrEmpty() {
+	for kSize, cLevelsByType := range plotCount {
+		for cLevel, plotCountByType := range cLevelsByType {
+			s.plotCount.WithLabelValues(fmt.Sprintf("%d", kSize), "og", fmt.Sprintf("%d", cLevel)).Set(float64(plotCountByType[PlotTypeOg]))
+			s.plotCount.WithLabelValues(fmt.Sprintf("%d", kSize), "pool", fmt.Sprintf("%d", cLevel)).Set(float64(plotCountByType[PlotTypePool]))
+		}
+	}
+
+	totalPlotCount := len(plots.Plots.OrEmpty())
+	s.totalPlots.Set(float64(totalPlotCount))
+
+	s.totalPlotsValue = uint64(totalPlotCount)
+}
+
+// PlotSizeCountHelper returns information about plot sizes and counts for the given set of plots
+// Return is (plotSize, plotCount)
+func PlotSizeCountHelper(plots []protocols.Plot) (map[uint8]map[uint8]map[PlotType]uint64, map[uint8]map[uint8]map[PlotType]uint64) {
+	// First, iterate through all the plots to get totals for each ksize
+	//          map[ksize]map[clevel]map[PlotType]uint64
+	plotSize := map[uint8]map[uint8]map[PlotType]uint64{}
+	plotCount := map[uint8]map[uint8]map[PlotType]uint64{}
+
+	for _, plot := range plots {
+		cLevel := plot.CompressionLevel.OrElse(uint8(0))
 		kSize := plot.Size
 
 		if _, ok := plotSize[kSize]; !ok {
-			plotSize[kSize] = map[plotType]uint64{
-				plotTypeOg:   0,
-				plotTypePool: 0,
-			}
+			// It's safe to assume that if plotSize isn't set, plotCount isn't either, since they are created together
+			plotSize[kSize] = map[uint8]map[PlotType]uint64{}
+			plotCount[kSize] = map[uint8]map[PlotType]uint64{}
 		}
 
-		if _, ok := plotCount[kSize]; !ok {
-			plotCount[kSize] = map[plotType]uint64{
-				plotTypeOg:   0,
-				plotTypePool: 0,
+		if _, ok := plotSize[kSize][cLevel]; !ok {
+			plotSize[kSize][cLevel] = map[PlotType]uint64{
+				PlotTypeOg:   0,
+				PlotTypePool: 0,
+			}
+			plotCount[kSize][cLevel] = map[PlotType]uint64{
+				PlotTypeOg:   0,
+				PlotTypePool: 0,
 			}
 		}
 
 		if plot.PoolContractPuzzleHash.IsPresent() {
-			plotSize[kSize][plotTypePool] += plot.FileSize
-			plotCount[kSize][plotTypePool]++
+			plotSize[kSize][cLevel][PlotTypePool] += plot.FileSize
+			plotCount[kSize][cLevel][PlotTypePool]++
 		} else {
-			plotSize[kSize][plotTypeOg] += plot.FileSize
-			plotCount[kSize][plotTypeOg]++
+			plotSize[kSize][cLevel][PlotTypeOg] += plot.FileSize
+			plotCount[kSize][cLevel][PlotTypeOg]++
 		}
 	}
 
-	// Now we can set the gauges with the calculated total values
-	for kSize, fileSizes := range plotSize {
-		s.plotFilesize.WithLabelValues(fmt.Sprintf("%d", kSize), "og").Set(float64(fileSizes[plotTypeOg]))
-		s.plotFilesize.WithLabelValues(fmt.Sprintf("%d", kSize), "pool").Set(float64(fileSizes[plotTypePool]))
-	}
-
-	for kSize, plotCountByType := range plotCount {
-		s.plotCount.WithLabelValues(fmt.Sprintf("%d", kSize), "og").Set(float64(plotCountByType[plotTypeOg]))
-		s.plotCount.WithLabelValues(fmt.Sprintf("%d", kSize), "pool").Set(float64(plotCountByType[plotTypePool]))
-	}
-
-	totalPlotCount := len(plots.Plots.OrEmpty())
-	s.totalPlots.Set(float64(totalPlotCount))
-
-	s.totalPlotsValue = uint64(totalPlotCount)
+	return plotSize, plotCount
 }