From 49fffe0340e61ef848f520072b55e6a0167d45ed Mon Sep 17 00:00:00 2001 From: Chris Marslender Date: Mon, 21 Aug 2023 14:30:11 -0500 Subject: [PATCH 1/5] Add compression level to plot metrics. Gather plot metrics from all harvesters via the farmer as an alternative to running this on every harvester --- go.mod | 2 +- go.sum | 2 + internal/metrics/farmer.go | 35 +++++++++++++ internal/metrics/harvester.go | 98 ++++++++++++++++++++++------------- 4 files changed, 100 insertions(+), 37 deletions(-) diff --git a/go.mod b/go.mod index 66138ce..b0d65f0 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/chia-network/chia-exporter go 1.18 require ( - github.com/chia-network/go-chia-libs v0.4.0 + github.com/chia-network/go-chia-libs v0.4.1-0.20230821182757-8e649e503882 github.com/chia-network/go-modules v0.0.4 github.com/oschwald/maxminddb-golang v1.12.0 github.com/prometheus/client_golang v1.16.0 diff --git a/go.sum b/go.sum index 116dcbd..82e1b71 100644 --- a/go.sum +++ b/go.sum @@ -45,6 +45,8 @@ github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chia-network/go-chia-libs v0.4.0 h1:KmBIJAtniqaFqyB+IKLPxEEKEtx89kBgP3tC4lgIT80= github.com/chia-network/go-chia-libs v0.4.0/go.mod h1:cLTizmlrAoyfL+PGLS5G7MT+Q288wtUHCneIDrIp7Mc= +github.com/chia-network/go-chia-libs v0.4.1-0.20230821182757-8e649e503882 h1:4aP3hjM1YgWMLDqLql0t9vRdL3IVVxLGwJ7D799EpEg= +github.com/chia-network/go-chia-libs v0.4.1-0.20230821182757-8e649e503882/go.mod h1:cLTizmlrAoyfL+PGLS5G7MT+Q288wtUHCneIDrIp7Mc= github.com/chia-network/go-modules v0.0.4 h1:XlCcuT4j1krLvsFT1Y49Un5xORwcTc8jjE4SHih7OTI= github.com/chia-network/go-modules v0.0.4/go.mod h1:JP8mG/9ieE76VcGIbzD5G3/4YDmvNhRryiQwp8GQr1U= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= diff --git a/internal/metrics/farmer.go b/internal/metrics/farmer.go index 0c8725c..1788e27 100644 --- a/internal/metrics/farmer.go +++ b/internal/metrics/farmer.go @@ -2,6 +2,7 @@ package metrics import ( "encoding/json" + "fmt" "time" "github.com/chia-network/go-chia-libs/pkg/rpc" @@ -31,6 +32,10 @@ type FarmerServiceMetrics struct { // Proof Metrics proofsFound *wrappedPrometheus.LazyCounter + + // Remote Harvester Plot Counts + plotFilesize *prometheus.GaugeVec + plotCount *prometheus.GaugeVec } // InitMetrics sets all the metrics properties @@ -46,6 +51,11 @@ func (s *FarmerServiceMetrics) InitMetrics() { // Proof Metrics s.proofsFound = s.metrics.newCounter(chiaServiceFarmer, "proofs_found", "Number of proofs found since the exporter has been running") + + // Remote harvester plot counts + plotLabels := []string{"host", "size", "type", "compression"} + s.plotFilesize = s.metrics.newGaugeVec(chiaServiceFarmer, "plot_filesize", "Filesize of plots separated by harvester", plotLabels) + s.plotCount = s.metrics.newGaugeVec(chiaServiceFarmer, "plot_count", "Number of plots separated by harvester", plotLabels) } // InitialData is called on startup of the metrics server, to allow seeding metrics with current/initial data @@ -86,6 +96,31 @@ func (s *FarmerServiceMetrics) ReceiveResponse(resp *types.WebsocketResponse) { // GetConnections handler for get_connections events func (s *FarmerServiceMetrics) GetConnections(resp *types.WebsocketResponse) { connectionCountHelper(resp, s.connectionCount) + harvesters, _, err := s.metrics.httpClient.FarmerService.GetHarvesters(&rpc.FarmerGetHarvestersOptions{}) + if err != nil { + log.Errorf("farmer: Error getting harvesters: %s\n", err.Error()) + return + } + + for _, harvester := range harvesters.Harvesters { + plotSize, plotCount := PlotSizeCountHelper(harvester.Plots) + + // Now we can set the gauges with the calculated total values + // Labels: "host", "size", "type", "compression" + for kSize, cLevels := range plotSize { + for cLevel, fileSizes := range cLevels { + s.plotFilesize.WithLabelValues(harvester.Connection.Host, fmt.Sprintf("%d", kSize), "og", fmt.Sprintf("%d", cLevel)).Set(float64(fileSizes[PlotTypeOg])) + s.plotFilesize.WithLabelValues(harvester.Connection.Host, fmt.Sprintf("%d", kSize), "pool", fmt.Sprintf("%d", cLevel)).Set(float64(fileSizes[PlotTypePool])) + } + } + + for kSize, cLevelsByType := range plotCount { + for cLevel, plotCountByType := range cLevelsByType { + s.plotCount.WithLabelValues(harvester.Connection.Host, fmt.Sprintf("%d", kSize), "og", fmt.Sprintf("%d", cLevel)).Set(float64(plotCountByType[PlotTypeOg])) + s.plotCount.WithLabelValues(harvester.Connection.Host, fmt.Sprintf("%d", kSize), "pool", fmt.Sprintf("%d", cLevel)).Set(float64(plotCountByType[PlotTypePool])) + } + } + } } // SubmittedPartial handles a received submitted_partial event diff --git a/internal/metrics/harvester.go b/internal/metrics/harvester.go index d45ed4c..b04fd81 100644 --- a/internal/metrics/harvester.go +++ b/internal/metrics/harvester.go @@ -5,6 +5,7 @@ import ( "fmt" "time" + "github.com/chia-network/go-chia-libs/pkg/protocols" "github.com/chia-network/go-chia-libs/pkg/rpc" "github.com/chia-network/go-chia-libs/pkg/types" "github.com/prometheus/client_golang/prometheus" @@ -44,9 +45,10 @@ func (s *HarvesterServiceMetrics) InitMetrics() { // Connection Metrics s.connectionCount = s.metrics.newGaugeVec(chiaServiceHarvester, "connection_count", "Number of active connections for each type of peer", []string{"node_type"}) + plotLabels := []string{"size", "type", "compression"} s.totalPlots = s.metrics.newGauge(chiaServiceHarvester, "total_plots", "Total number of plots on this harvester") - s.plotFilesize = s.metrics.newGaugeVec(chiaServiceHarvester, "plot_filesize", "Total filesize of plots on this harvester, by K size", []string{"size", "type"}) - s.plotCount = s.metrics.newGaugeVec(chiaServiceHarvester, "plot_count", "Total count of plots on this harvester, by K size", []string{"size", "type"}) + s.plotFilesize = s.metrics.newGaugeVec(chiaServiceHarvester, "plot_filesize", "Total filesize of plots on this harvester, by K size", plotLabels) + s.plotCount = s.metrics.newGaugeVec(chiaServiceHarvester, "plot_count", "Total count of plots on this harvester, by K size", plotLabels) s.totalFoundProofs = s.metrics.newCounter(chiaServiceHarvester, "total_found_proofs", "Counter of total found proofs since the exporter started") s.lastFoundProofs = s.metrics.newGauge(chiaServiceHarvester, "last_found_proofs", "Number of proofs found for the last farmer_info event") @@ -155,55 +157,79 @@ func (s *HarvesterServiceMetrics) GetPlots(resp *types.WebsocketResponse) { s.ProcessGetPlots(plots) } +// PlotType is the type of plot (og or pool) +type PlotType uint8 + +const ( + // PlotTypeOg is the original plot format, no plotNFT + PlotTypeOg = PlotType(0) + // PlotTypePool is the new plotNFT plot format + PlotTypePool = PlotType(1) +) + // ProcessGetPlots processes the `GetPlotsResponse` from get_plots so that we can use this with websockets or HTTP RPC requests func (s *HarvesterServiceMetrics) ProcessGetPlots(plots *rpc.HarvesterGetPlotsResponse) { - // First, iterate through all the plots to get totals for each ksize - type plotType uint8 - plotTypeOg := plotType(0) - plotTypePool := plotType(1) + plotSize, plotCount := PlotSizeCountHelper(plots.Plots.OrEmpty()) - plotSize := map[uint8]map[plotType]uint64{} - plotCount := map[uint8]map[plotType]uint64{} + // Now we can set the gauges with the calculated total values + // Labels: "size", "type", "compression" + for kSize, cLevels := range plotSize { + for cLevel, fileSizes := range cLevels { + s.plotFilesize.WithLabelValues(fmt.Sprintf("%d", kSize), "og", fmt.Sprintf("%d", cLevel)).Set(float64(fileSizes[PlotTypeOg])) + s.plotFilesize.WithLabelValues(fmt.Sprintf("%d", kSize), "pool", fmt.Sprintf("%d", cLevel)).Set(float64(fileSizes[PlotTypePool])) + } + } - for _, plot := range plots.Plots.OrEmpty() { + for kSize, cLevelsByType := range plotCount { + for cLevel, plotCountByType := range cLevelsByType { + s.plotCount.WithLabelValues(fmt.Sprintf("%d", kSize), "og", fmt.Sprintf("%d", cLevel)).Set(float64(plotCountByType[PlotTypeOg])) + s.plotCount.WithLabelValues(fmt.Sprintf("%d", kSize), "pool", fmt.Sprintf("%d", cLevel)).Set(float64(plotCountByType[PlotTypePool])) + } + } + + totalPlotCount := len(plots.Plots.OrEmpty()) + s.totalPlots.Set(float64(totalPlotCount)) + + s.totalPlotsValue = uint64(totalPlotCount) +} + +// PlotSizeCountHelper returns information about plot sizes and counts for the given set of plots +// Return is (plotSize, plotCount) +func PlotSizeCountHelper(plots []protocols.Plot) (map[uint8]map[uint8]map[PlotType]uint64, map[uint8]map[uint8]map[PlotType]uint64) { + // First, iterate through all the plots to get totals for each ksize + // map[ksize]map[clevel]map[PlotType]uint64 + plotSize := map[uint8]map[uint8]map[PlotType]uint64{} + plotCount := map[uint8]map[uint8]map[PlotType]uint64{} + + for _, plot := range plots { + cLevel := plot.CompressionLevel.OrElse(uint8(0)) kSize := plot.Size if _, ok := plotSize[kSize]; !ok { - plotSize[kSize] = map[plotType]uint64{ - plotTypeOg: 0, - plotTypePool: 0, - } + // It's safe to assume that if plotSize isn't set, plotCount isn't either, since they are created together + plotSize[kSize] = map[uint8]map[PlotType]uint64{} + plotCount[kSize] = map[uint8]map[PlotType]uint64{} } - if _, ok := plotCount[kSize]; !ok { - plotCount[kSize] = map[plotType]uint64{ - plotTypeOg: 0, - plotTypePool: 0, + if _, ok := plotSize[kSize][cLevel]; !ok { + plotSize[kSize][cLevel] = map[PlotType]uint64{ + PlotTypeOg: 0, + PlotTypePool: 0, + } + plotCount[kSize][cLevel] = map[PlotType]uint64{ + PlotTypeOg: 0, + PlotTypePool: 0, } } if plot.PoolContractPuzzleHash.IsPresent() { - plotSize[kSize][plotTypePool] += plot.FileSize - plotCount[kSize][plotTypePool]++ + plotSize[kSize][cLevel][PlotTypePool] += plot.FileSize + plotCount[kSize][cLevel][PlotTypePool]++ } else { - plotSize[kSize][plotTypeOg] += plot.FileSize - plotCount[kSize][plotTypeOg]++ + plotSize[kSize][cLevel][PlotTypeOg] += plot.FileSize + plotCount[kSize][cLevel][PlotTypeOg]++ } } - // Now we can set the gauges with the calculated total values - for kSize, fileSizes := range plotSize { - s.plotFilesize.WithLabelValues(fmt.Sprintf("%d", kSize), "og").Set(float64(fileSizes[plotTypeOg])) - s.plotFilesize.WithLabelValues(fmt.Sprintf("%d", kSize), "pool").Set(float64(fileSizes[plotTypePool])) - } - - for kSize, plotCountByType := range plotCount { - s.plotCount.WithLabelValues(fmt.Sprintf("%d", kSize), "og").Set(float64(plotCountByType[plotTypeOg])) - s.plotCount.WithLabelValues(fmt.Sprintf("%d", kSize), "pool").Set(float64(plotCountByType[plotTypePool])) - } - - totalPlotCount := len(plots.Plots.OrEmpty()) - s.totalPlots.Set(float64(totalPlotCount)) - - s.totalPlotsValue = uint64(totalPlotCount) + return plotSize, plotCount } From df6fc9aa6f1d251fd302f5e11ed6de55d1a14aef Mon Sep 17 00:00:00 2001 From: Chris Marslender Date: Mon, 21 Aug 2023 14:39:53 -0500 Subject: [PATCH 2/5] Update to latest go-chia-libs --- go.mod | 2 +- go.sum | 6 ++---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index b0d65f0..42a00bb 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/chia-network/chia-exporter go 1.18 require ( - github.com/chia-network/go-chia-libs v0.4.1-0.20230821182757-8e649e503882 + github.com/chia-network/go-chia-libs v0.4.1-0.20230821193838-4062eee42a67 github.com/chia-network/go-modules v0.0.4 github.com/oschwald/maxminddb-golang v1.12.0 github.com/prometheus/client_golang v1.16.0 diff --git a/go.sum b/go.sum index 82e1b71..3873ddb 100644 --- a/go.sum +++ b/go.sum @@ -43,10 +43,8 @@ github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6r github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/chia-network/go-chia-libs v0.4.0 h1:KmBIJAtniqaFqyB+IKLPxEEKEtx89kBgP3tC4lgIT80= -github.com/chia-network/go-chia-libs v0.4.0/go.mod h1:cLTizmlrAoyfL+PGLS5G7MT+Q288wtUHCneIDrIp7Mc= -github.com/chia-network/go-chia-libs v0.4.1-0.20230821182757-8e649e503882 h1:4aP3hjM1YgWMLDqLql0t9vRdL3IVVxLGwJ7D799EpEg= -github.com/chia-network/go-chia-libs v0.4.1-0.20230821182757-8e649e503882/go.mod h1:cLTizmlrAoyfL+PGLS5G7MT+Q288wtUHCneIDrIp7Mc= +github.com/chia-network/go-chia-libs v0.4.1-0.20230821193838-4062eee42a67 h1:JBvlhAFzx9SPQZ75i+c1OZE+admL8eSNeSz7zKsLlOA= +github.com/chia-network/go-chia-libs v0.4.1-0.20230821193838-4062eee42a67/go.mod h1:cLTizmlrAoyfL+PGLS5G7MT+Q288wtUHCneIDrIp7Mc= github.com/chia-network/go-modules v0.0.4 h1:XlCcuT4j1krLvsFT1Y49Un5xORwcTc8jjE4SHih7OTI= github.com/chia-network/go-modules v0.0.4/go.mod h1:JP8mG/9ieE76VcGIbzD5G3/4YDmvNhRryiQwp8GQr1U= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= From c2e2d49092ff50765917a148683bc4d78586ac78 Mon Sep 17 00:00:00 2001 From: Chris Marslender Date: Mon, 21 Aug 2023 15:35:11 -0500 Subject: [PATCH 3/5] Add remaining remote harvester metrics surfaced from the farmer --- go.mod | 2 +- go.sum | 2 + internal/metrics/farmer.go | 93 +++++++++++++++++++++++++++++++------- 3 files changed, 79 insertions(+), 18 deletions(-) diff --git a/go.mod b/go.mod index 42a00bb..0a2957b 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/chia-network/chia-exporter go 1.18 require ( - github.com/chia-network/go-chia-libs v0.4.1-0.20230821193838-4062eee42a67 + github.com/chia-network/go-chia-libs v0.4.1-0.20230821202324-4025c23838ec github.com/chia-network/go-modules v0.0.4 github.com/oschwald/maxminddb-golang v1.12.0 github.com/prometheus/client_golang v1.16.0 diff --git a/go.sum b/go.sum index 3873ddb..46c2e76 100644 --- a/go.sum +++ b/go.sum @@ -45,6 +45,8 @@ github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chia-network/go-chia-libs v0.4.1-0.20230821193838-4062eee42a67 h1:JBvlhAFzx9SPQZ75i+c1OZE+admL8eSNeSz7zKsLlOA= github.com/chia-network/go-chia-libs v0.4.1-0.20230821193838-4062eee42a67/go.mod h1:cLTizmlrAoyfL+PGLS5G7MT+Q288wtUHCneIDrIp7Mc= +github.com/chia-network/go-chia-libs v0.4.1-0.20230821202324-4025c23838ec h1:oq6sevwBWQ4t/HUMSZV8Gwjx63+V/AYMO6BSHzLDcz0= +github.com/chia-network/go-chia-libs v0.4.1-0.20230821202324-4025c23838ec/go.mod h1:cLTizmlrAoyfL+PGLS5G7MT+Q288wtUHCneIDrIp7Mc= github.com/chia-network/go-modules v0.0.4 h1:XlCcuT4j1krLvsFT1Y49Un5xORwcTc8jjE4SHih7OTI= github.com/chia-network/go-modules v0.0.4/go.mod h1:JP8mG/9ieE76VcGIbzD5G3/4YDmvNhRryiQwp8GQr1U= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= diff --git a/internal/metrics/farmer.go b/internal/metrics/farmer.go index 1788e27..591a296 100644 --- a/internal/metrics/farmer.go +++ b/internal/metrics/farmer.go @@ -3,7 +3,6 @@ package metrics import ( "encoding/json" "fmt" - "time" "github.com/chia-network/go-chia-libs/pkg/rpc" "github.com/chia-network/go-chia-libs/pkg/types" @@ -25,6 +24,13 @@ type FarmerServiceMetrics struct { // Connection Metrics connectionCount *prometheus.GaugeVec + // Keep a local copy of the plot count, so we can do other actions when the value changes + // Tracked per node ID, since we get checkins from all harvesters here + totalPlotsValue map[types.Bytes32]uint64 + + // Also have to keep track of node id to hostname mapping, since not all responses have the friendly hostname + nodeIDToHostname map[types.Bytes32]string + // Partial/Pooling Metrics submittedPartials *prometheus.CounterVec currentDifficulty *prometheus.GaugeVec @@ -34,12 +40,20 @@ type FarmerServiceMetrics struct { proofsFound *wrappedPrometheus.LazyCounter // Remote Harvester Plot Counts - plotFilesize *prometheus.GaugeVec - plotCount *prometheus.GaugeVec + plotFilesize *prometheus.GaugeVec + plotCount *prometheus.GaugeVec + totalFoundProofs *prometheus.CounterVec + lastFoundProofs *prometheus.GaugeVec + totalEligiblePlots *prometheus.CounterVec + lastEligiblePlots *prometheus.GaugeVec + lastLookupTime *prometheus.GaugeVec } // InitMetrics sets all the metrics properties func (s *FarmerServiceMetrics) InitMetrics() { + s.totalPlotsValue = map[types.Bytes32]uint64{} + s.nodeIDToHostname = map[types.Bytes32]string{} + // Connection Metrics s.connectionCount = s.metrics.newGaugeVec(chiaServiceFarmer, "connection_count", "Number of active connections for each type of peer", []string{"node_type"}) @@ -53,23 +67,23 @@ func (s *FarmerServiceMetrics) InitMetrics() { s.proofsFound = s.metrics.newCounter(chiaServiceFarmer, "proofs_found", "Number of proofs found since the exporter has been running") // Remote harvester plot counts - plotLabels := []string{"host", "size", "type", "compression"} + plotLabels := []string{"host", "node_id", "size", "type", "compression"} s.plotFilesize = s.metrics.newGaugeVec(chiaServiceFarmer, "plot_filesize", "Filesize of plots separated by harvester", plotLabels) s.plotCount = s.metrics.newGaugeVec(chiaServiceFarmer, "plot_count", "Number of plots separated by harvester", plotLabels) + s.totalFoundProofs = s.metrics.newCounterVec(chiaServiceFarmer, "total_found_proofs", "Counter of total found proofs since the exporter started", []string{"host", "node_id"}) + s.lastFoundProofs = s.metrics.newGaugeVec(chiaServiceFarmer, "last_found_proofs", "Number of proofs found for the last farmer_info event", []string{"host", "node_id"}) + s.totalEligiblePlots = s.metrics.newCounterVec(chiaServiceFarmer, "total_eligible_plots", "Counter of total eligible plots since the exporter started", []string{"host", "node_id"}) + s.lastEligiblePlots = s.metrics.newGaugeVec(chiaServiceFarmer, "last_eligible_plots", "Number of eligible plots for the last farmer_info event", []string{"host", "node_id"}) + s.lastLookupTime = s.metrics.newGaugeVec(chiaServiceFarmer, "last_lookup_time", "Lookup time for the last farmer_info event", []string{"host", "node_id"}) } // InitialData is called on startup of the metrics server, to allow seeding metrics with current/initial data -func (s *FarmerServiceMetrics) InitialData() {} +func (s *FarmerServiceMetrics) InitialData() { + utils.LogErr(s.metrics.client.FarmerService.GetConnections(&rpc.GetConnectionsOptions{})) +} // SetupPollingMetrics starts any metrics that happen on an interval -func (s *FarmerServiceMetrics) SetupPollingMetrics() { - go func() { - for { - utils.LogErr(s.metrics.client.FarmerService.GetConnections(&rpc.GetConnectionsOptions{})) - time.Sleep(15 * time.Second) - } - }() -} +func (s *FarmerServiceMetrics) SetupPollingMetrics() {} // Disconnected clears/unregisters metrics when the connection drops func (s *FarmerServiceMetrics) Disconnected() { @@ -86,10 +100,18 @@ func (s *FarmerServiceMetrics) ReceiveResponse(resp *types.WebsocketResponse) { switch resp.Command { case "get_connections": s.GetConnections(resp) + case "new_farming_info": + s.NewFarmingInfo(resp) case "submitted_partial": s.SubmittedPartial(resp) case "proof": s.Proof(resp) + case "harvester_removed": + fallthrough + case "add_connection": + fallthrough + case "close_connection": + utils.LogErr(s.metrics.client.FarmerService.GetConnections(&rpc.GetConnectionsOptions{})) } } @@ -103,24 +125,61 @@ func (s *FarmerServiceMetrics) GetConnections(resp *types.WebsocketResponse) { } for _, harvester := range harvesters.Harvesters { + // keep track of the node ID to host mapping + s.nodeIDToHostname[harvester.Connection.NodeID] = harvester.Connection.Host + + _totalPlotCount := uint64(0) plotSize, plotCount := PlotSizeCountHelper(harvester.Plots) // Now we can set the gauges with the calculated total values // Labels: "host", "size", "type", "compression" for kSize, cLevels := range plotSize { for cLevel, fileSizes := range cLevels { - s.plotFilesize.WithLabelValues(harvester.Connection.Host, fmt.Sprintf("%d", kSize), "og", fmt.Sprintf("%d", cLevel)).Set(float64(fileSizes[PlotTypeOg])) - s.plotFilesize.WithLabelValues(harvester.Connection.Host, fmt.Sprintf("%d", kSize), "pool", fmt.Sprintf("%d", cLevel)).Set(float64(fileSizes[PlotTypePool])) + s.plotFilesize.WithLabelValues(harvester.Connection.Host, harvester.Connection.NodeID.String(), fmt.Sprintf("%d", kSize), "og", fmt.Sprintf("%d", cLevel)).Set(float64(fileSizes[PlotTypeOg])) + s.plotFilesize.WithLabelValues(harvester.Connection.Host, harvester.Connection.NodeID.String(), fmt.Sprintf("%d", kSize), "pool", fmt.Sprintf("%d", cLevel)).Set(float64(fileSizes[PlotTypePool])) } } for kSize, cLevelsByType := range plotCount { for cLevel, plotCountByType := range cLevelsByType { - s.plotCount.WithLabelValues(harvester.Connection.Host, fmt.Sprintf("%d", kSize), "og", fmt.Sprintf("%d", cLevel)).Set(float64(plotCountByType[PlotTypeOg])) - s.plotCount.WithLabelValues(harvester.Connection.Host, fmt.Sprintf("%d", kSize), "pool", fmt.Sprintf("%d", cLevel)).Set(float64(plotCountByType[PlotTypePool])) + _totalPlotCount += plotCountByType[PlotTypeOg] + _totalPlotCount += plotCountByType[PlotTypePool] + + s.plotCount.WithLabelValues(harvester.Connection.Host, harvester.Connection.NodeID.String(), fmt.Sprintf("%d", kSize), "og", fmt.Sprintf("%d", cLevel)).Set(float64(plotCountByType[PlotTypeOg])) + s.plotCount.WithLabelValues(harvester.Connection.Host, harvester.Connection.NodeID.String(), fmt.Sprintf("%d", kSize), "pool", fmt.Sprintf("%d", cLevel)).Set(float64(plotCountByType[PlotTypePool])) } } + + s.totalPlotsValue[harvester.Connection.NodeID] = _totalPlotCount + } +} + +// NewFarmingInfo handles new_farming_info events +func (s *FarmerServiceMetrics) NewFarmingInfo(resp *types.WebsocketResponse) { + info := &types.EventFarmerNewFarmingInfo{} + err := json.Unmarshal(resp.Data, info) + if err != nil { + log.Errorf("Error unmarshalling: %s\n", err.Error()) + return } + + nodeID := info.FarmingInfo.NodeID + hostname, foundHostname := s.nodeIDToHostname[nodeID] + if !foundHostname || s.totalPlotsValue[nodeID] != uint64(info.FarmingInfo.TotalPlots) { + log.Debugf("Missing node ID to host mapping or plot count doesn't match. Refreshing harvester info. New Plot Count: %d | Previous Plot Count: %d\n", info.FarmingInfo.TotalPlots, s.totalPlotsValue) + // When plot counts change, we have to refresh information about the plots + utils.LogErr(s.metrics.client.FarmerService.GetConnections(&rpc.GetConnectionsOptions{})) + } + + if foundHostname { + // Labels: "host", "node_id" + s.totalFoundProofs.WithLabelValues(hostname, nodeID.String()).Add(float64(info.FarmingInfo.Proofs)) + s.lastFoundProofs.WithLabelValues(hostname, nodeID.String()).Set(float64(info.FarmingInfo.Proofs)) + s.totalEligiblePlots.WithLabelValues(hostname, nodeID.String()).Add(float64(info.FarmingInfo.PassedFilter)) + s.lastEligiblePlots.WithLabelValues(hostname, nodeID.String()).Set(float64(info.FarmingInfo.PassedFilter)) + s.lastLookupTime.WithLabelValues(hostname, nodeID.String()).Set(float64(info.FarmingInfo.LookupTime)) + } + } // SubmittedPartial handles a received submitted_partial event From 0a004eb29f5b85dfea8514db570b423b952735f7 Mon Sep 17 00:00:00 2001 From: Chris Marslender Date: Mon, 21 Aug 2023 15:45:47 -0500 Subject: [PATCH 4/5] Reset plot counters before updating and on disconnect --- internal/metrics/farmer.go | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/internal/metrics/farmer.go b/internal/metrics/farmer.go index 591a296..3b3c4dd 100644 --- a/internal/metrics/farmer.go +++ b/internal/metrics/farmer.go @@ -88,6 +88,11 @@ func (s *FarmerServiceMetrics) SetupPollingMetrics() {} // Disconnected clears/unregisters metrics when the connection drops func (s *FarmerServiceMetrics) Disconnected() { s.connectionCount.Reset() + s.plotFilesize.Reset() + s.plotCount.Reset() + s.lastFoundProofs.Reset() + s.lastEligiblePlots.Reset() + s.lastLookupTime.Reset() } // Reconnected is called when the service is reconnected after the websocket was disconnected @@ -124,6 +129,10 @@ func (s *FarmerServiceMetrics) GetConnections(resp *types.WebsocketResponse) { return } + // Must be reset prior to setting new values in case all of a particular type, k-size, or c level are gone + s.plotFilesize.Reset() + s.plotCount.Reset() + for _, harvester := range harvesters.Harvesters { // keep track of the node ID to host mapping s.nodeIDToHostname[harvester.Connection.NodeID] = harvester.Connection.Host @@ -166,7 +175,7 @@ func (s *FarmerServiceMetrics) NewFarmingInfo(resp *types.WebsocketResponse) { nodeID := info.FarmingInfo.NodeID hostname, foundHostname := s.nodeIDToHostname[nodeID] if !foundHostname || s.totalPlotsValue[nodeID] != uint64(info.FarmingInfo.TotalPlots) { - log.Debugf("Missing node ID to host mapping or plot count doesn't match. Refreshing harvester info. New Plot Count: %d | Previous Plot Count: %d\n", info.FarmingInfo.TotalPlots, s.totalPlotsValue) + log.Debugf("Missing node ID to host mapping or plot count doesn't match. Refreshing harvester info. New Plot Count: %d | Previous Plot Count: %d\n", info.FarmingInfo.TotalPlots, s.totalPlotsValue[nodeID]) // When plot counts change, we have to refresh information about the plots utils.LogErr(s.metrics.client.FarmerService.GetConnections(&rpc.GetConnectionsOptions{})) } From f13896b86d92bb97d06d67136f5d1b2f706f35b0 Mon Sep 17 00:00:00 2001 From: Chris Marslender Date: Mon, 21 Aug 2023 15:51:56 -0500 Subject: [PATCH 5/5] go-chia-libs v0.5.0 --- go.mod | 2 +- go.sum | 6 ++---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index 0a2957b..283cc5d 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/chia-network/chia-exporter go 1.18 require ( - github.com/chia-network/go-chia-libs v0.4.1-0.20230821202324-4025c23838ec + github.com/chia-network/go-chia-libs v0.5.0 github.com/chia-network/go-modules v0.0.4 github.com/oschwald/maxminddb-golang v1.12.0 github.com/prometheus/client_golang v1.16.0 diff --git a/go.sum b/go.sum index 46c2e76..69ffaa8 100644 --- a/go.sum +++ b/go.sum @@ -43,10 +43,8 @@ github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6r github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/chia-network/go-chia-libs v0.4.1-0.20230821193838-4062eee42a67 h1:JBvlhAFzx9SPQZ75i+c1OZE+admL8eSNeSz7zKsLlOA= -github.com/chia-network/go-chia-libs v0.4.1-0.20230821193838-4062eee42a67/go.mod h1:cLTizmlrAoyfL+PGLS5G7MT+Q288wtUHCneIDrIp7Mc= -github.com/chia-network/go-chia-libs v0.4.1-0.20230821202324-4025c23838ec h1:oq6sevwBWQ4t/HUMSZV8Gwjx63+V/AYMO6BSHzLDcz0= -github.com/chia-network/go-chia-libs v0.4.1-0.20230821202324-4025c23838ec/go.mod h1:cLTizmlrAoyfL+PGLS5G7MT+Q288wtUHCneIDrIp7Mc= +github.com/chia-network/go-chia-libs v0.5.0 h1:j4skq96iOIawGWqXebk87HBYJlUZbBIhh9zr/LLGxKs= +github.com/chia-network/go-chia-libs v0.5.0/go.mod h1:cLTizmlrAoyfL+PGLS5G7MT+Q288wtUHCneIDrIp7Mc= github.com/chia-network/go-modules v0.0.4 h1:XlCcuT4j1krLvsFT1Y49Un5xORwcTc8jjE4SHih7OTI= github.com/chia-network/go-modules v0.0.4/go.mod h1:JP8mG/9ieE76VcGIbzD5G3/4YDmvNhRryiQwp8GQr1U= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=