diff --git a/go.mod b/go.mod index 66138ce..b0d65f0 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/chia-network/chia-exporter go 1.18 require ( - github.com/chia-network/go-chia-libs v0.4.0 + github.com/chia-network/go-chia-libs v0.4.1-0.20230821182757-8e649e503882 github.com/chia-network/go-modules v0.0.4 github.com/oschwald/maxminddb-golang v1.12.0 github.com/prometheus/client_golang v1.16.0 diff --git a/go.sum b/go.sum index 116dcbd..82e1b71 100644 --- a/go.sum +++ b/go.sum @@ -45,6 +45,8 @@ github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chia-network/go-chia-libs v0.4.0 h1:KmBIJAtniqaFqyB+IKLPxEEKEtx89kBgP3tC4lgIT80= github.com/chia-network/go-chia-libs v0.4.0/go.mod h1:cLTizmlrAoyfL+PGLS5G7MT+Q288wtUHCneIDrIp7Mc= +github.com/chia-network/go-chia-libs v0.4.1-0.20230821182757-8e649e503882 h1:4aP3hjM1YgWMLDqLql0t9vRdL3IVVxLGwJ7D799EpEg= +github.com/chia-network/go-chia-libs v0.4.1-0.20230821182757-8e649e503882/go.mod h1:cLTizmlrAoyfL+PGLS5G7MT+Q288wtUHCneIDrIp7Mc= github.com/chia-network/go-modules v0.0.4 h1:XlCcuT4j1krLvsFT1Y49Un5xORwcTc8jjE4SHih7OTI= github.com/chia-network/go-modules v0.0.4/go.mod h1:JP8mG/9ieE76VcGIbzD5G3/4YDmvNhRryiQwp8GQr1U= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= diff --git a/internal/metrics/farmer.go b/internal/metrics/farmer.go index 0c8725c..1788e27 100644 --- a/internal/metrics/farmer.go +++ b/internal/metrics/farmer.go @@ -2,6 +2,7 @@ package metrics import ( "encoding/json" + "fmt" "time" "github.com/chia-network/go-chia-libs/pkg/rpc" @@ -31,6 +32,10 @@ type FarmerServiceMetrics struct { // Proof Metrics proofsFound *wrappedPrometheus.LazyCounter + + // Remote Harvester Plot Counts + plotFilesize *prometheus.GaugeVec + plotCount *prometheus.GaugeVec } // InitMetrics sets all the metrics properties @@ -46,6 +51,11 @@ func (s *FarmerServiceMetrics) InitMetrics() { // Proof Metrics s.proofsFound = s.metrics.newCounter(chiaServiceFarmer, "proofs_found", "Number of proofs found since the exporter has been running") + + // Remote harvester plot counts + plotLabels := []string{"host", "size", "type", "compression"} + s.plotFilesize = s.metrics.newGaugeVec(chiaServiceFarmer, "plot_filesize", "Filesize of plots separated by harvester", plotLabels) + s.plotCount = s.metrics.newGaugeVec(chiaServiceFarmer, "plot_count", "Number of plots separated by harvester", plotLabels) } // InitialData is called on startup of the metrics server, to allow seeding metrics with current/initial data @@ -86,6 +96,31 @@ func (s *FarmerServiceMetrics) ReceiveResponse(resp *types.WebsocketResponse) { // GetConnections handler for get_connections events func (s *FarmerServiceMetrics) GetConnections(resp *types.WebsocketResponse) { connectionCountHelper(resp, s.connectionCount) + harvesters, _, err := s.metrics.httpClient.FarmerService.GetHarvesters(&rpc.FarmerGetHarvestersOptions{}) + if err != nil { + log.Errorf("farmer: Error getting harvesters: %s\n", err.Error()) + return + } + + for _, harvester := range harvesters.Harvesters { + plotSize, plotCount := PlotSizeCountHelper(harvester.Plots) + + // Now we can set the gauges with the calculated total values + // Labels: "host", "size", "type", "compression" + for kSize, cLevels := range plotSize { + for cLevel, fileSizes := range cLevels { + s.plotFilesize.WithLabelValues(harvester.Connection.Host, fmt.Sprintf("%d", kSize), "og", fmt.Sprintf("%d", cLevel)).Set(float64(fileSizes[PlotTypeOg])) + s.plotFilesize.WithLabelValues(harvester.Connection.Host, fmt.Sprintf("%d", kSize), "pool", fmt.Sprintf("%d", cLevel)).Set(float64(fileSizes[PlotTypePool])) + } + } + + for kSize, cLevelsByType := range plotCount { + for cLevel, plotCountByType := range cLevelsByType { + s.plotCount.WithLabelValues(harvester.Connection.Host, fmt.Sprintf("%d", kSize), "og", fmt.Sprintf("%d", cLevel)).Set(float64(plotCountByType[PlotTypeOg])) + s.plotCount.WithLabelValues(harvester.Connection.Host, fmt.Sprintf("%d", kSize), "pool", fmt.Sprintf("%d", cLevel)).Set(float64(plotCountByType[PlotTypePool])) + } + } + } } // SubmittedPartial handles a received submitted_partial event diff --git a/internal/metrics/harvester.go b/internal/metrics/harvester.go index d45ed4c..b04fd81 100644 --- a/internal/metrics/harvester.go +++ b/internal/metrics/harvester.go @@ -5,6 +5,7 @@ import ( "fmt" "time" + "github.com/chia-network/go-chia-libs/pkg/protocols" "github.com/chia-network/go-chia-libs/pkg/rpc" "github.com/chia-network/go-chia-libs/pkg/types" "github.com/prometheus/client_golang/prometheus" @@ -44,9 +45,10 @@ func (s *HarvesterServiceMetrics) InitMetrics() { // Connection Metrics s.connectionCount = s.metrics.newGaugeVec(chiaServiceHarvester, "connection_count", "Number of active connections for each type of peer", []string{"node_type"}) + plotLabels := []string{"size", "type", "compression"} s.totalPlots = s.metrics.newGauge(chiaServiceHarvester, "total_plots", "Total number of plots on this harvester") - s.plotFilesize = s.metrics.newGaugeVec(chiaServiceHarvester, "plot_filesize", "Total filesize of plots on this harvester, by K size", []string{"size", "type"}) - s.plotCount = s.metrics.newGaugeVec(chiaServiceHarvester, "plot_count", "Total count of plots on this harvester, by K size", []string{"size", "type"}) + s.plotFilesize = s.metrics.newGaugeVec(chiaServiceHarvester, "plot_filesize", "Total filesize of plots on this harvester, by K size", plotLabels) + s.plotCount = s.metrics.newGaugeVec(chiaServiceHarvester, "plot_count", "Total count of plots on this harvester, by K size", plotLabels) s.totalFoundProofs = s.metrics.newCounter(chiaServiceHarvester, "total_found_proofs", "Counter of total found proofs since the exporter started") s.lastFoundProofs = s.metrics.newGauge(chiaServiceHarvester, "last_found_proofs", "Number of proofs found for the last farmer_info event") @@ -155,55 +157,79 @@ func (s *HarvesterServiceMetrics) GetPlots(resp *types.WebsocketResponse) { s.ProcessGetPlots(plots) } +// PlotType is the type of plot (og or pool) +type PlotType uint8 + +const ( + // PlotTypeOg is the original plot format, no plotNFT + PlotTypeOg = PlotType(0) + // PlotTypePool is the new plotNFT plot format + PlotTypePool = PlotType(1) +) + // ProcessGetPlots processes the `GetPlotsResponse` from get_plots so that we can use this with websockets or HTTP RPC requests func (s *HarvesterServiceMetrics) ProcessGetPlots(plots *rpc.HarvesterGetPlotsResponse) { - // First, iterate through all the plots to get totals for each ksize - type plotType uint8 - plotTypeOg := plotType(0) - plotTypePool := plotType(1) + plotSize, plotCount := PlotSizeCountHelper(plots.Plots.OrEmpty()) - plotSize := map[uint8]map[plotType]uint64{} - plotCount := map[uint8]map[plotType]uint64{} + // Now we can set the gauges with the calculated total values + // Labels: "size", "type", "compression" + for kSize, cLevels := range plotSize { + for cLevel, fileSizes := range cLevels { + s.plotFilesize.WithLabelValues(fmt.Sprintf("%d", kSize), "og", fmt.Sprintf("%d", cLevel)).Set(float64(fileSizes[PlotTypeOg])) + s.plotFilesize.WithLabelValues(fmt.Sprintf("%d", kSize), "pool", fmt.Sprintf("%d", cLevel)).Set(float64(fileSizes[PlotTypePool])) + } + } - for _, plot := range plots.Plots.OrEmpty() { + for kSize, cLevelsByType := range plotCount { + for cLevel, plotCountByType := range cLevelsByType { + s.plotCount.WithLabelValues(fmt.Sprintf("%d", kSize), "og", fmt.Sprintf("%d", cLevel)).Set(float64(plotCountByType[PlotTypeOg])) + s.plotCount.WithLabelValues(fmt.Sprintf("%d", kSize), "pool", fmt.Sprintf("%d", cLevel)).Set(float64(plotCountByType[PlotTypePool])) + } + } + + totalPlotCount := len(plots.Plots.OrEmpty()) + s.totalPlots.Set(float64(totalPlotCount)) + + s.totalPlotsValue = uint64(totalPlotCount) +} + +// PlotSizeCountHelper returns information about plot sizes and counts for the given set of plots +// Return is (plotSize, plotCount) +func PlotSizeCountHelper(plots []protocols.Plot) (map[uint8]map[uint8]map[PlotType]uint64, map[uint8]map[uint8]map[PlotType]uint64) { + // First, iterate through all the plots to get totals for each ksize + // map[ksize]map[clevel]map[PlotType]uint64 + plotSize := map[uint8]map[uint8]map[PlotType]uint64{} + plotCount := map[uint8]map[uint8]map[PlotType]uint64{} + + for _, plot := range plots { + cLevel := plot.CompressionLevel.OrElse(uint8(0)) kSize := plot.Size if _, ok := plotSize[kSize]; !ok { - plotSize[kSize] = map[plotType]uint64{ - plotTypeOg: 0, - plotTypePool: 0, - } + // It's safe to assume that if plotSize isn't set, plotCount isn't either, since they are created together + plotSize[kSize] = map[uint8]map[PlotType]uint64{} + plotCount[kSize] = map[uint8]map[PlotType]uint64{} } - if _, ok := plotCount[kSize]; !ok { - plotCount[kSize] = map[plotType]uint64{ - plotTypeOg: 0, - plotTypePool: 0, + if _, ok := plotSize[kSize][cLevel]; !ok { + plotSize[kSize][cLevel] = map[PlotType]uint64{ + PlotTypeOg: 0, + PlotTypePool: 0, + } + plotCount[kSize][cLevel] = map[PlotType]uint64{ + PlotTypeOg: 0, + PlotTypePool: 0, } } if plot.PoolContractPuzzleHash.IsPresent() { - plotSize[kSize][plotTypePool] += plot.FileSize - plotCount[kSize][plotTypePool]++ + plotSize[kSize][cLevel][PlotTypePool] += plot.FileSize + plotCount[kSize][cLevel][PlotTypePool]++ } else { - plotSize[kSize][plotTypeOg] += plot.FileSize - plotCount[kSize][plotTypeOg]++ + plotSize[kSize][cLevel][PlotTypeOg] += plot.FileSize + plotCount[kSize][cLevel][PlotTypeOg]++ } } - // Now we can set the gauges with the calculated total values - for kSize, fileSizes := range plotSize { - s.plotFilesize.WithLabelValues(fmt.Sprintf("%d", kSize), "og").Set(float64(fileSizes[plotTypeOg])) - s.plotFilesize.WithLabelValues(fmt.Sprintf("%d", kSize), "pool").Set(float64(fileSizes[plotTypePool])) - } - - for kSize, plotCountByType := range plotCount { - s.plotCount.WithLabelValues(fmt.Sprintf("%d", kSize), "og").Set(float64(plotCountByType[plotTypeOg])) - s.plotCount.WithLabelValues(fmt.Sprintf("%d", kSize), "pool").Set(float64(plotCountByType[plotTypePool])) - } - - totalPlotCount := len(plots.Plots.OrEmpty()) - s.totalPlots.Set(float64(totalPlotCount)) - - s.totalPlotsValue = uint64(totalPlotCount) + return plotSize, plotCount }